This is an automated email from the ASF dual-hosted git repository.
zykkk pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a7d31f0c32d [improvement](jdbc catalog) Change the connection pool
parameters of Jdbc Catalog to be configurable (#28779)
a7d31f0c32d is described below
commit a7d31f0c32d78d7f98988aa9f55603d9fc7cb3bd
Author: zy-kkk <zhongy...@gmail.com>
AuthorDate: Wed Dec 27 17:05:14 2023 +0800
[improvement](jdbc catalog) Change the connection pool parameters of Jdbc
Catalog to be configurable (#28779)
---
bin/start_be.sh | 2 +-
conf/be.conf | 4 ++--
.../java/org/apache/doris/jdbc/JdbcExecutor.java | 6 +++++-
.../main/java/org/apache/doris/common/Config.java | 20 ++++++++++++++++++++
.../doris/datasource/jdbc/client/JdbcClient.java | 14 ++++++++------
5 files changed, 36 insertions(+), 10 deletions(-)
diff --git a/bin/start_be.sh b/bin/start_be.sh
index 48ae202d9fa..67a2e7d2201 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -303,7 +303,7 @@ java_version="$(
CUR_DATE=$(date +%Y%m%d-%H%M%S)
LOG_PATH="-DlogPath=${DORIS_HOME}/log/jni.log"
COMMON_OPTS="-Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
-JDBC_OPTS="-DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000
-DJDBC_MAX_WAIT_TIME=5000"
+JDBC_OPTS="-DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000
-DJDBC_MAX_WAIT_TIME=5000 -DJDBC_KEEP_ALIVE=FALSE"
if [[ "${java_version}" -gt 8 ]]; then
if [[ -z ${JAVA_OPTS_FOR_JDK_9} ]]; then
diff --git a/conf/be.conf b/conf/be.conf
index 98371325914..9c877901d59 100644
--- a/conf/be.conf
+++ b/conf/be.conf
@@ -19,10 +19,10 @@ CUR_DATE=`date +%Y%m%d-%H%M%S`
PPROF_TMPDIR="$DORIS_HOME/log/"
-JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE
-XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100
-DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
+JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE
-XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100
-DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000 -DJDBC_KEEP_ALIVE=FALSE"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE
-XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100
-DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
+JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE
-XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100
-DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000 -DJDBC_KEEP_ALIVE=FALSE"
# since 1.2, the JAVA_HOME need to be set to run BE process.
# JAVA_HOME=/path/to/jdk/
diff --git
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
index c5625717c0e..d3f4006723d 100644
---
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
+++
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
@@ -90,6 +90,7 @@ public class JdbcExecutor {
private int minIdleSize;
private int maxIdleTime;
private int maxWaitTime;
+ private boolean isKeepAlive;
private TOdbcTableType tableType;
public JdbcExecutor(byte[] thriftParams) throws Exception {
@@ -105,12 +106,14 @@ public class JdbcExecutor {
maxPoolSize = Integer.valueOf(System.getProperty("JDBC_MAX_POOL",
"100"));
maxIdleTime = Integer.valueOf(System.getProperty("JDBC_MAX_IDLE_TIME",
"300000"));
maxWaitTime = Integer.valueOf(System.getProperty("JDBC_MAX_WAIT_TIME",
"5000"));
+ isKeepAlive = Boolean.valueOf(System.getProperty("JDBC_KEEP_ALIVE",
"false"));
minIdleSize = minPoolSize > 0 ? 1 : 0;
LOG.info("JdbcExecutor set minPoolSize = " + minPoolSize
+ ", maxPoolSize = " + maxPoolSize
+ ", maxIdleTime = " + maxIdleTime
+ ", maxWaitTime = " + maxWaitTime
- + ", minIdleSize = " + minIdleSize);
+ + ", minIdleSize = " + minIdleSize
+ + ", isKeepAlive = " + isKeepAlive);
init(request.driver_path, request.statement, request.batch_size,
request.jdbc_driver_class,
request.jdbc_url, request.jdbc_user, request.jdbc_password,
request.op, request.table_type);
}
@@ -441,6 +444,7 @@ public class JdbcExecutor {
setValidationQuery(ds, tableType);
ds.setTimeBetweenEvictionRunsMillis(maxIdleTime /
5);
ds.setMinEvictableIdleTimeMillis(maxIdleTime);
+ ds.setKeepAlive(isKeepAlive);
druidDataSource = ds;
// here is a cache of datasource, which using the
string(jdbcUrl + jdbcUser +
// jdbcPassword) as key.
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 907d02ffc26..979f49e1fa5 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -140,6 +140,26 @@ public class Config extends ConfigBase {
+ "if the specified driver file path is not an absolute
path, Doris will find jars from this path"})
public static String jdbc_drivers_dir = System.getenv("DORIS_HOME") +
"/jdbc_drivers";
+ @ConfField(description = {"JDBC Catalog FE 连接池的最小连接数",
+ "The minimum number of JDBC Catalog FE connection pool"})
+ public static int jdbc_min_pool_size = 1;
+
+ @ConfField(description = {"JDBC Catalog FE 连接池的最大连接数",
+ "The maximum number of JDBC Catalog FE connection pool"})
+ public static int jdbc_max_pool_size = 100;
+
+ @ConfField(description = {"JDBC Catalog FE 连接池的最大空闲连接时间",
+ "The maximum idle time of JDBC Catalog FE connection pool"})
+ public static int jdbc_max_idle_time = 300000;
+
+ @ConfField(description = {"JDBC Catalog FE 连接池的最大等待时间",
+ "The maximum wait time of JDBC Catalog FE connection pool"})
+ public static int jdbc_max_wait_time = 5000;
+
+ @ConfField(description = {"JDBC Catalog FE 连接池的 KeepAlive 策略",
+ "The keep alive strategy of JDBC Catalog FE connection pool"})
+ public static boolean jdbc_keep_alive = false;
+
@ConfField(mutable = true, masterOnly = true, description = {"broker load
时，单个节点上 load 执行计划的默认并行度",
"The default parallelism of the load execution plan on a single
node when the broker load is submitted"})
public static int default_load_parallelism = 1;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
index 11969cb8654..bdf5a6eab38 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.JdbcResource;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.Util;
@@ -131,17 +132,18 @@ public abstract class JdbcClient {
dataSource.setUrl(jdbcUrl);
dataSource.setUsername(jdbcUser);
dataSource.setPassword(password);
- dataSource.setMinIdle(1);
- dataSource.setInitialSize(1);
- dataSource.setMaxActive(100);
- dataSource.setTimeBetweenEvictionRunsMillis(600000);
- dataSource.setMinEvictableIdleTimeMillis(300000);
+ dataSource.setMinIdle(Config.jdbc_min_pool_size > 0 ? 1 : 0);
+ dataSource.setInitialSize(Config.jdbc_min_pool_size);
+ dataSource.setMaxActive(Config.jdbc_max_pool_size);
+
dataSource.setTimeBetweenEvictionRunsMillis(Config.jdbc_max_idle_time * 2L);
+
dataSource.setMinEvictableIdleTimeMillis(Config.jdbc_max_idle_time);
// set connection timeout to 5s.
// The default is 30s, which is too long.
// Because when querying information_schema db, BE will call
thrift rpc(default timeout is 30s)
// to FE to get schema info, and may create connection here, if we
set it too long and the url is invalid,
// it may cause the thrift rpc timeout.
- dataSource.setMaxWait(5000);
+ dataSource.setMaxWait(Config.jdbc_max_wait_time);
+ dataSource.setKeepAlive(Config.jdbc_keep_alive);
} catch (MalformedURLException e) {
throw new JdbcClientException("MalformedURLException to load class
about " + driverUrl, e);
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
