This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2868ad45d01e34474b24ba9da5879b9db00e4b2d
Author: zy-kkk <zhongy...@gmail.com>
AuthorDate: Fri Aug 23 14:08:19 2024 +0800
[fix](jdbc catalog) Fix Memory Leak by Enabling Weak References in HikariCP
(#39582)
This PR addresses a memory leak issue caused by FastList objects in
HikariCP being retained by ThreadLocal variables, which are not easily
garbage collected in long-running JNI threads. To mitigate this, a
system property com.zaxxer.hikari.useWeakReferences is set to true,
ensuring that WeakReference is used for ThreadLocal objects, allowing
the garbage collector to reclaim memory more effectively.
Even though setting this will affect some performance, solving resource
leaks is relatively more important
Performance difference before and after setting
Before setting:
10 concurrency 0.02-0.05
100 concurrency 0.18-0.4
After setting:
10 concurrency 0.02-0.07
100 concurrency 0.18-0.7
---
.../src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java | 3 ++-
.../src/main/java/org/apache/doris/jdbc/JdbcDataSource.java | 6 ++++--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
index b25294021ee..23ab66f5763 100644
---
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
+++
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
@@ -94,6 +94,7 @@ public abstract class BaseJdbcExecutor implements
JdbcExecutor {
.setConnectionPoolMaxLifeTime(request.connection_pool_max_life_time)
.setConnectionPoolKeepAlive(request.connection_pool_keep_alive);
JdbcDataSource.getDataSource().setCleanupInterval(request.connection_pool_cache_clear_time);
+ System.setProperty("com.zaxxer.hikari.useWeakReferences", "true");
init(config, request.statement);
this.jdbcDriverVersion = getJdbcDriverVersion();
}
@@ -322,7 +323,7 @@ public abstract class BaseJdbcExecutor implements
JdbcExecutor {
ds.setKeepaliveTime(config.getConnectionPoolMaxLifeTime() / 5L); // default 6
min
}
hikariDataSource = ds;
-
JdbcDataSource.getDataSource().putSource(hikariDataSourceKey, ds);
+
JdbcDataSource.getDataSource().putSource(hikariDataSourceKey, hikariDataSource);
LOG.info("JdbcClient set"
+ " ConnectionPoolMinSize = " +
config.getConnectionPoolMinSize()
+ ", ConnectionPoolMaxSize = " +
config.getConnectionPoolMaxSize()
diff --git
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
index 447566596a7..1aea763852f 100644
---
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
+++
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
@@ -59,8 +59,10 @@ public class JdbcDataSource {
}
public void setCleanupInterval(long interval) {
- this.cleanupInterval = interval * 1000L;
- restartCleanupTask();
+ if (this.cleanupInterval != interval * 1000L) {
+ this.cleanupInterval = interval * 1000L;
+ restartCleanupTask();
+ }
}
private synchronized void restartCleanupTask() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
