This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 641c49d925d7c6d06a72f5dd359e9b5f397b5f5f
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Wed Jan 18 17:48:06 2023 +0800
[fix](jdbc) fix jdbc driver bug and external datasource p2 test case issue
(#16033)
Fix bug that when create jdbc resource with only jdbc driver file name, it
will failed to do checksum
This is because we forgot the pass the full driver url to JdbcClient.
Add ResultSet.FETCH_FORWARD and set AutoCommit to false to jdbc connection,
so to avoid OOM when fetching large amount of data
set useCursorFetch in jdbc url for both MySQL and PostgreSQL.
Fix some p2 external datasource bug
---
.../main/java/org/apache/doris/catalog/JdbcResource.java | 8 +++++---
.../java/org/apache/doris/external/jdbc/JdbcClient.java | 13 ++-----------
.../src/main/java/org/apache/doris/udf/JdbcExecutor.java | 5 ++++-
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
index add3009de8..afdd099702 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
@@ -189,10 +189,10 @@ public class JdbcResource extends Resource {
// skip checking checksum when running ut
return "";
}
- String fullDriverPath = getRealDriverPath(driverPath);
+ String fullDriverUrl = getFullDriverUrl(driverPath);
InputStream inputStream = null;
try {
- inputStream = Util.getInputStreamFromUrl(fullDriverPath, null,
HTTP_TIMEOUT_MS, HTTP_TIMEOUT_MS);
+ inputStream = Util.getInputStreamFromUrl(fullDriverUrl, null,
HTTP_TIMEOUT_MS, HTTP_TIMEOUT_MS);
MessageDigest digest = MessageDigest.getInstance("MD5");
byte[] buf = new byte[4096];
int bytesRead = 0;
@@ -213,7 +213,7 @@ public class JdbcResource extends Resource {
}
}
- private static String getRealDriverPath(String driverUrl) {
+ public static String getFullDriverUrl(String driverUrl) {
try {
URI uri = new URI(driverUrl);
String schema = uri.getScheme();
@@ -254,6 +254,8 @@ public class JdbcResource extends Resource {
// it will convert to Doris tinyint, not bit.
newJdbcUrl = checkJdbcUrlParam(newJdbcUrl, "yearIsDateType",
"true", "false");
newJdbcUrl = checkJdbcUrlParam(newJdbcUrl, "tinyInt1isBit",
"true", "false");
+ }
+ if (dbType.equals(MYSQL) || dbType.equals(POSTGRESQL)) {
newJdbcUrl = checkJdbcUrlParam(newJdbcUrl, "useCursorFetch",
"false", "true");
}
return newJdbcUrl;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
index cc8fc80b3d..6f420246c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
@@ -50,29 +50,20 @@ public class JdbcClient {
private String dbType;
private String jdbcUser;
- private String jdbcPasswd;
- private String jdbcUrl;
- private String driverUrl;
- private String driverClass;
private URLClassLoader classLoader = null;
private HikariDataSource dataSource = null;
-
public JdbcClient(String user, String password, String jdbcUrl, String
driverUrl, String driverClass) {
this.jdbcUser = user;
- this.jdbcPasswd = password;
- this.jdbcUrl = jdbcUrl;
this.dbType = parseDbType(jdbcUrl);
- this.driverUrl = driverUrl;
- this.driverClass = driverClass;
ClassLoader oldClassLoader =
Thread.currentThread().getContextClassLoader();
try {
// TODO(ftw): The problem here is that the jar package is handled
by FE
// and URLClassLoader may load the jar package directly into
memory
- URL[] urls = {new URL(driverUrl)};
+ URL[] urls = {new URL(JdbcResource.getFullDriverUrl(driverUrl))};
// set parent ClassLoader to null, we can achieve class loading
isolation.
classLoader = URLClassLoader.newInstance(urls, null);
Thread.currentThread().setContextClassLoader(classLoader);
@@ -80,7 +71,7 @@ public class JdbcClient {
config.setDriverClassName(driverClass);
config.setJdbcUrl(jdbcUrl);
config.setUsername(jdbcUser);
- config.setPassword(jdbcPasswd);
+ config.setPassword(password);
config.setMaximumPoolSize(1);
dataSource = new HikariDataSource(config);
} catch (MalformedURLException e) {
diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
index f7273e3718..7f2366b948 100644
--- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
+++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
@@ -249,8 +249,10 @@ public class JdbcExecutor {
dataSource = new HikariDataSource(config);
conn = dataSource.getConnection();
if (op == TJdbcOperation.READ) {
+ conn.setAutoCommit(false);
Preconditions.checkArgument(sql != null);
- stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
+ stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY,
+ ResultSet.FETCH_FORWARD);
stmt.setFetchSize(batchSize);
} else {
stmt = conn.createStatement();
@@ -264,3 +266,4 @@ public class JdbcExecutor {
}
}
}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
