This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 6caa3cf761a branch-3.0: [opt](jdbc catalog) Compatible with higher
ClickHouse JDBC Driver versions #46026 (#48182)
6caa3cf761a is described below
commit 6caa3cf761af8897ce7f51d759a56d9c7e8fd14e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Feb 22 14:10:23 2025 +0800
branch-3.0: [opt](jdbc catalog) Compatible with higher ClickHouse JDBC
Driver versions #46026 (#48182)
Cherry-picked from #46026
Co-authored-by: zy-kkk <zhongy...@gmail.com>
---
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 2 +-
.../jdbc/client/JdbcClickHouseClient.java | 130 +++++++++++++++++++++
.../doris/datasource/jdbc/client/JdbcClient.java | 12 ++
.../jdbc/client/JdbcOceanBaseClient.java | 2 +-
.../jdbc/client/JdbcClickHouseClientTest.java | 67 +++++++++++
.../jdbc/{ => client}/JdbcClientExceptionTest.java | 4 +-
.../jdbc/test_clickhouse_jdbc_catalog.out | Bin 4381 -> 8056 bytes
.../jdbc/test_clickhouse_jdbc_catalog.groovy | 73 ++++++++----
8 files changed, 261 insertions(+), 29 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
index fac322d21eb..03554dafbcb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
@@ -382,7 +382,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
jdbcClient.testConnection();
} catch (JdbcClientException e) {
String errorMessage = "Test FE Connection to JDBC Failed: " +
e.getMessage();
- LOG.error(errorMessage, e);
+ LOG.warn(errorMessage, e);
throw new DdlException(errorMessage, e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
index bdf0cbbc934..4f340bebed4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
@@ -22,12 +22,103 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
+import com.google.common.collect.Lists;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
import java.util.Optional;
+import java.util.function.Consumer;
public class JdbcClickHouseClient extends JdbcClient {
+ private final Boolean databaseTermIsCatalog;
+
protected JdbcClickHouseClient(JdbcClientConfig jdbcClientConfig) {
super(jdbcClientConfig);
+ try (Connection conn = getConnection()) {
+ String jdbcUrl = conn.getMetaData().getURL();
+ if (!isNewClickHouseDriver(getJdbcDriverVersion())) {
+ this.databaseTermIsCatalog = false;
+ } else {
+ this.databaseTermIsCatalog =
"catalog".equalsIgnoreCase(getDatabaseTermFromUrl(jdbcUrl));
+ }
+ } catch (SQLException e) {
+ throw new JdbcClientException("Failed to initialize
JdbcClickHouseClient: %s", e.getMessage());
+ }
+ }
+
+ @Override
+ public List<String> getDatabaseNameList() {
+ Connection conn = null;
+ ResultSet rs = null;
+ List<String> remoteDatabaseNames = Lists.newArrayList();
+ try {
+ conn = getConnection();
+ if (isOnlySpecifiedDatabase && includeDatabaseMap.isEmpty() &&
excludeDatabaseMap.isEmpty()) {
+ if (databaseTermIsCatalog) {
+ remoteDatabaseNames.add(conn.getCatalog());
+ } else {
+ remoteDatabaseNames.add(conn.getSchema());
+ }
+ } else {
+ if (databaseTermIsCatalog) {
+ rs = conn.getMetaData().getCatalogs();
+ } else {
+ rs = conn.getMetaData().getSchemas(conn.getCatalog(),
null);
+ }
+ while (rs.next()) {
+ remoteDatabaseNames.add(rs.getString(1));
+ }
+ }
+ } catch (SQLException e) {
+ throw new JdbcClientException("failed to get database name list
from jdbc", e);
+ } finally {
+ close(rs, conn);
+ }
+ return filterDatabaseNames(remoteDatabaseNames);
+ }
+
+ @Override
+ protected void processTable(String remoteDbName, String remoteTableName,
String[] tableTypes,
+ Consumer<ResultSet> resultSetConsumer) {
+ Connection conn = null;
+ ResultSet rs = null;
+ try {
+ conn = super.getConnection();
+ DatabaseMetaData databaseMetaData = conn.getMetaData();
+ if (databaseTermIsCatalog) {
+ rs = databaseMetaData.getTables(remoteDbName, null,
remoteTableName, tableTypes);
+ } else {
+ rs = databaseMetaData.getTables(null, remoteDbName,
remoteTableName, tableTypes);
+ }
+ resultSetConsumer.accept(rs);
+ } catch (SQLException e) {
+ throw new JdbcClientException("Failed to process table", e);
+ } finally {
+ close(rs, conn);
+ }
+ }
+
+ @Override
+ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData,
String catalogName, String remoteDbName,
+ String remoteTableName) throws SQLException {
+ if (databaseTermIsCatalog) {
+ return databaseMetaData.getColumns(remoteDbName, null,
remoteTableName, null);
+ } else {
+ return databaseMetaData.getColumns(catalogName, remoteDbName,
remoteTableName, null);
+ }
+ }
+
+ @Override
+ protected String getCatalogName(Connection conn) throws SQLException {
+ if (databaseTermIsCatalog) {
+ return null;
+ } else {
+ return conn.getCatalog();
+ }
}
@Override
@@ -121,4 +212,43 @@ public class JdbcClickHouseClient extends JdbcClient {
return Type.UNSUPPORTED;
}
}
+
+ /**
+ * Determine whether the driver version is greater than or equal to 0.5.0.
+ */
+ private static boolean isNewClickHouseDriver(String driverVersion) {
+ if (driverVersion == null) {
+ throw new JdbcClientException("Driver version cannot be null");
+ }
+ try {
+ String[] versionParts = driverVersion.split("\\.");
+ int majorVersion = Integer.parseInt(versionParts[0]);
+ int minorVersion = Integer.parseInt(versionParts[1]);
+ // Determine whether it is greater than or equal to 0.5.x
+ return (majorVersion > 0) || (majorVersion == 0 && minorVersion >=
5);
+ } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+ throw new JdbcClientException("Invalid clickhouse driver version
format: " + driverVersion, e);
+ }
+ }
+
+ /**
+ * Extract databaseterm parameters from the jdbc url.
+ */
+ private String getDatabaseTermFromUrl(String jdbcUrl) {
+ if (jdbcUrl != null &&
jdbcUrl.toLowerCase().contains("databaseterm=schema")) {
+ return "schema";
+ }
+ return "catalog";
+ }
+
+ /**
+ * Get the driver version.
+ */
+ public String getJdbcDriverVersion() {
+ try (Connection conn = getConnection()) {
+ return conn.getMetaData().getDriverVersion();
+ } catch (SQLException e) {
+ throw new JdbcClientException("Failed to get jdbc driver version",
e);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
index a3dfdcda319..121f7d6ba04 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
@@ -480,4 +480,16 @@ public abstract class JdbcClient {
public String getTestQuery() {
return "select 1";
}
+
+ public String getJdbcDriverVersion() {
+ Connection conn = null;
+ try {
+ conn = getConnection();
+ return conn.getMetaData().getDriverVersion();
+ } catch (SQLException e) {
+ throw new JdbcClientException("Failed to get jdbc driver version",
e);
+ } finally {
+ close(conn);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
index 0d3970c774b..f43119875d6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
@@ -54,7 +54,7 @@ public class JdbcOceanBaseClient extends JdbcClient {
throw new JdbcClientException("Failed to determine OceanBase
compatibility mode");
}
} catch (SQLException e) {
- throw new JdbcClientException("Failed to initialize
JdbcOceanBaseClient", e.getMessage());
+ throw new JdbcClientException("Failed to initialize
JdbcOceanBaseClient: %s", e.getMessage());
} finally {
close(rs, stmt, conn);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
new file mode 100644
index 00000000000..99e4aa62dd5
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.jdbc.client;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Method;
+
+public class JdbcClickHouseClientTest {
+
+ @Test
+ public void testIsNewClickHouseDriver() {
+ try {
+ Method method =
JdbcClickHouseClient.class.getDeclaredMethod("isNewClickHouseDriver",
String.class);
+ method.setAccessible(true);
+
+ // Valid test cases
+ Assert.assertTrue((boolean) method.invoke(null, "0.5.0")); //
Major version 0, Minor version 5
+ Assert.assertTrue((boolean) method.invoke(null, "1.0.0")); //
Major version 1
+ Assert.assertTrue((boolean) method.invoke(null, "0.6.3 (revision:
a6a8a22)")); // Major version 0, Minor version 6
+ Assert.assertFalse((boolean) method.invoke(null, "0.4.2 (revision:
1513b27)")); // Major version 0, Minor version 4
+
+ // Invalid version formats
+ try {
+ method.invoke(null, "invalid.version"); // Invalid version
format
+ Assert.fail("Expected JdbcClientException for invalid version
'invalid.version'");
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof JdbcClientException);
+ Assert.assertTrue(e.getCause().getMessage().contains("Invalid
clickhouse driver version format"));
+ }
+
+ try {
+ method.invoke(null, ""); // Empty version
+ Assert.fail("Expected JdbcClientException for empty version");
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof JdbcClientException);
+ Assert.assertTrue(e.getCause().getMessage().contains("Invalid
clickhouse driver version format"));
+ }
+
+ try {
+ method.invoke(null, (Object) null); // Null version
+ Assert.fail("Expected JdbcClientException for null version");
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof JdbcClientException);
+ Assert.assertTrue(e.getCause().getMessage().contains("Driver
version cannot be null"));
+ }
+ } catch (Exception e) {
+ Assert.fail("Exception occurred while testing
isNewClickHouseDriver: " + e.getMessage());
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcClientExceptionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClientExceptionTest.java
similarity index 97%
rename from
fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcClientExceptionTest.java
rename to
fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClientExceptionTest.java
index 1bbf54e9438..c99f2bcfe26 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcClientExceptionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClientExceptionTest.java
@@ -15,9 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.datasource.jdbc;
-
-import org.apache.doris.datasource.jdbc.client.JdbcClientException;
+package org.apache.doris.datasource.jdbc.client;
import org.junit.Assert;
import org.junit.Test;
diff --git
a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out
b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out
index 22f85579a83..910338e0862 100644
Binary files
a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out
and
b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out
differ
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
index 7875ac4244e..3e625596d99 100644
---
a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
+++
b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
@@ -26,6 +26,7 @@ suite("test_clickhouse_jdbc_catalog",
"p0,external,clickhouse,external_docker,ex
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.4.2-all.jar";
+ String driver_url_7 =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.7.1-patch1-all.jar";
String inDorisTable = "test_clickhouse_jdbc_doris_in_tb";
@@ -91,33 +92,57 @@ suite("test_clickhouse_jdbc_catalog",
"p0,external,clickhouse,external_docker,ex
order_qt_dt_with_tz """ select * from dt_with_tz order by id; """
- sql """create catalog if not exists
clickhouse_catalog_test_conn_correct properties(
+ sql """ drop catalog if exists ${catalog_name} """
+
+
+ sql """ drop catalog if exists clickhouse_7_default """
+ sql """ create catalog if not exists clickhouse_7_default properties(
"type"="jdbc",
"user"="default",
"password"="123456",
"jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test",
- "driver_url" = "${driver_url}",
- "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver",
- "test_connection" = "true"
- );
- """
- order_qt_test_conn_correct """ select * from
clickhouse_catalog_test_conn_correct.doris_test.type; """
-
- test {
- sql """create catalog if not exists
clickhouse_catalog_test_conn_mistake properties(
- "type"="jdbc",
- "user"="default",
- "password"="1234567",
- "jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test",
- "driver_url" = "${driver_url}",
- "driver_class" =
"com.clickhouse.jdbc.ClickHouseDriver",
- "test_connection" = "true"
- );
- """
- exception "Test FE Connection to JDBC Failed"
- }
- sql """ drop catalog if exists ${catalog_name} """
- sql """ drop catalog if exists clickhouse_catalog_test_conn_correct """
- sql """ drop catalog if exists clickhouse_catalog_test_conn_mistake """
+ "driver_url" = "${driver_url_7}",
+ "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver"
+ );"""
+
+ order_qt_clickhouse_7_default """ select * from
clickhouse_7_default.doris_test.type; """
+ order_qt_clickhouse_7_default_tvf """ select * from query('catalog' =
'clickhouse_7_default', 'query' = 'select * from doris_test.type;') order by 1;
"""
+ order_qt_clickhouse_7_default_tvf_arr """ select * from
query('catalog' = 'clickhouse_7_default', 'query' = 'select * from
doris_test.arr;') order by 1; """
+
+ sql """ drop catalog if exists clickhouse_7_default """
+
+ sql """ drop catalog if exists clickhouse_7_catalog """
+
+ sql """ create catalog if not exists clickhouse_7_catalog properties(
+ "type"="jdbc",
+ "user"="default",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test?databaseTerm=catalog",
+ "driver_url" = "${driver_url_7}",
+ "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver"
+ );"""
+
+ order_qt_clickhouse_7_catalog """ select * from
clickhouse_7_catalog.doris_test.type; """
+ order_qt_clickhouse_7_catalog_tvf """ select * from query('catalog' =
'clickhouse_7_catalog', 'query' = 'select * from doris_test.type;') order by 1;
"""
+ order_qt_clickhouse_7_catalog_tvf_arr """ select * from
query('catalog' = 'clickhouse_7_catalog', 'query' = 'select * from
doris_test.arr;') order by 1; """
+
+ sql """ drop catalog if exists clickhouse_7_catalog """
+
+ sql """ drop catalog if exists clickhouse_7_schema """
+
+ sql """ create catalog if not exists clickhouse_7_schema properties(
+ "type"="jdbc",
+ "user"="default",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test?databaseTerm=schema",
+ "driver_url" = "${driver_url_7}",
+ "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver"
+ );"""
+
+ order_qt_clickhouse_7_schema """ select * from
clickhouse_7_schema.doris_test.type; """
+ order_qt_clickhouse_7_schema_tvf """ select * from query('catalog' =
'clickhouse_7_schema', 'query' = 'select * from doris_test.type;') order by 1;
"""
+ order_qt_clickhouse_7_schema_tvf_arr """ select * from query('catalog'
= 'clickhouse_7_schema', 'query' = 'select * from doris_test.arr;') order by 1;
"""
+
+ sql """ drop catalog if exists clickhouse_7_schema """
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
