This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3a8fbc9d44b [opt](catalog) use table in db object return get db
(#46211)
3a8fbc9d44b is described below
commit 3a8fbc9d44b0fc7ec5337d19506c6124653b13ab
Author: zy-kkk <zhongy...@gmail.com>
AuthorDate: Tue Dec 31 22:00:48 2024 +0800
[opt](catalog) use table in db object return get db (#46211)
### What problem does this PR solve?
This pull request includes several changes across multiple files to
improve logging, add missing logic, and correct SQL syntax. The most
important changes include adding logging for synchronization actions,
setting the database if missing, and correcting SQL syntax for MySQL
queries.
Co-authored-by: morningman <yun...@selectdb.com>
---
.../java/org/apache/doris/datasource/ExternalCatalog.java | 5 ++++-
.../java/org/apache/doris/datasource/ExternalDatabase.java | 14 ++++++++++++++
.../java/org/apache/doris/datasource/ExternalTable.java | 2 +-
.../apache/doris/datasource/jdbc/JdbcExternalTable.java | 2 +-
.../external_table_p0/hive/test_hive_use_meta_cache.out | 4 ++++
.../suites/external_table_p0/hive/test_hive_parquet.groovy | 3 ++-
.../external_table_p2/hudi/test_hudi_snapshot.groovy | 3 +--
7 files changed, 27 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 840baa3ac6b..e7fd443e9a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -110,7 +110,7 @@ public abstract class ExternalCatalog
public static final String CREATE_TIME = "create_time";
public static final boolean DEFAULT_USE_META_CACHE = true;
- public static final String FOUND_CONFLICTING = "Found conflicting";
+ public static final String FOUND_CONFLICTING = "Found conflicting";
public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES =
"only_test_lower_case_table_names";
// Properties that should not be shown in the `show create catalog` result
@@ -746,6 +746,7 @@ public abstract class ExternalCatalog
Preconditions.checkNotNull(db.get());
tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
tmpIdToDb.put(db.get().getId(), db.get());
+ LOG.info("Synchronized database (refresh): [Name: {}, ID: {}]",
db.get().getFullName(), db.get().getId());
}
for (int i = 0; i < log.getCreateCount(); i++) {
ExternalDatabase<? extends ExternalTable> db =
@@ -754,6 +755,8 @@ public abstract class ExternalCatalog
if (db != null) {
tmpDbNameToId.put(db.getFullName(), db.getId());
tmpIdToDb.put(db.getId(), db);
+ LOG.info("Synchronized database (create): [Name: {}, ID: {},
Remote Name: {}]",
+ db.getFullName(), db.getId(),
log.getRemoteDbNames().get(i));
}
}
dbNameToId = tmpDbNameToId;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 099d76e98e1..1759c3e30b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -211,6 +211,13 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
if (table.isPresent()) {
tmpTableNameToId.put(table.get().getName(),
table.get().getId());
tmpIdToTbl.put(table.get().getId(), table.get());
+
+ // Add logic to set the database if missing
+ if (table.get().getDb() == null) {
+ table.get().setDb(this);
+ }
+ LOG.info("Synchronized table (refresh): [Name: {}, ID: {}]",
table.get().getName(),
+ table.get().getId());
}
}
for (int i = 0; i < log.getCreateCount(); i++) {
@@ -219,6 +226,13 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
log.getCreateTableIds().get(i), catalog, this,
false);
tmpTableNameToId.put(table.getName(), table.getId());
tmpIdToTbl.put(table.getId(), table);
+
+ // Add logic to set the database if missing
+ if (table.getDb() == null) {
+ table.setDb(this);
+ }
+ LOG.info("Synchronized table (create): [Name: {}, ID: {}, Remote
Name: {}]",
+ table.getName(), table.getId(),
log.getRemoteTableNames().get(i));
}
tableNameToId = tmpTableNameToId;
idToTbl = tmpIdToTbl;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index dcad165afa2..b78d3f0dc65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -308,7 +308,7 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
@Override
public DatabaseIf getDatabase() {
- return catalog.getDbNullable(dbName);
+ return this.db;
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
index 36378e80ac3..426e75231f9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -51,7 +51,7 @@ import java.util.stream.Collectors;
public class JdbcExternalTable extends ExternalTable {
private static final Logger LOG =
LogManager.getLogger(JdbcExternalTable.class);
- public static final String MYSQL_ROW_COUNT_SQL = "SELECT max(row_count) as
rows FROM ("
+ public static final String MYSQL_ROW_COUNT_SQL = "SELECT max(row_count) as
`rows` FROM ("
+ "(SELECT TABLE_ROWS AS row_count FROM INFORMATION_SCHEMA.TABLES "
+ "WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = '${tblName}' "
+ "AND TABLE_TYPE = 'BASE TABLE') "
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
index 4d54c601c87..d8b269fdf0d 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
@@ -34,6 +34,8 @@ p1=part1
p1=part2
-- !sql10 --
+test_use_meta_cache_partitioned_tbl_hive
+test_use_meta_cache_tbl_hive
-- !sql11 --
@@ -123,6 +125,8 @@ p1=part1
p1=part2
-- !sql10 --
+test_use_meta_cache_partitioned_tbl_hive
+test_use_meta_cache_tbl_hive
-- !sql11 --
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
index d1e0922fe74..81d2954daf9 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
@@ -186,7 +186,8 @@ suite("test_hive_parquet",
"p0,external,hive,external_docker,external_docker_hiv
"type"="hms",
'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
);"""
- sql """use `${catalog_name}`.`default`"""
+ sql """switch ${catalog_name}"""
+ sql """use `default`"""
sql """set enable_fallback_to_original_planner=false;"""
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
index 89d89709b3c..0dd7321875b 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
@@ -34,7 +34,6 @@ suite("test_hudi_snapshot",
"p2,external,hudi,external_remote,external_remote_hu
sql """ use regression_hudi;"""
sql """ set enable_fallback_to_original_planner=false """
- // 创建groovy函数，接收table_name为参数
def test_hudi_snapshot_querys = { table_name ->
// Query users by event_time in descending order and limit output
qt_q01 """SELECT * FROM ${table_name} ORDER BY event_time DESC LIMIT
10;"""
@@ -49,7 +48,7 @@ suite("test_hudi_snapshot",
"p2,external,hudi,external_remote,external_remote_hu
qt_q04 """SELECT * FROM ${table_name} WHERE event_time BETWEEN
'2024-01-01 00:00:00' AND '2024-12-31 23:59:59' ORDER BY event_time LIMIT 10;"""
// Count users by age group and limit output
- qt_q05 """SELECT age, COUNT(*) AS user_count FROM ${table_name} GROUP
BY age ORDER BY user_count DESC LIMIT 5;"""
+ qt_q05 """SELECT age, COUNT(*) AS user_count FROM ${table_name} GROUP
BY age ORDER BY user_count, age DESC LIMIT 5;"""
// Query users with purchase records and limit output
qt_q06 """SELECT user_id, purchases FROM ${table_name} WHERE
array_size(purchases) > 0 ORDER BY user_id LIMIT 5;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
