This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ba28c9401d0 [fix](external catalog) Fixed non-master FE log replay
during refresh (#46369)
ba28c9401d0 is described below
commit ba28c9401d0fdf62babd584a889f3bab665a918c
Author: zy-kkk <zhongy...@gmail.com>
AuthorDate: Fri Jan 3 19:31:08 2025 +0800
[fix](external catalog) Fixed non-master FE log replay during refresh
(#46369)
In the case of multiple FEs, non-Master FEs create Catalogs by pulling
master FE logs for replay. However, when refreshing, there is no need to
determine the remote name to reset the meta information. It is only
determined when creating.
---
.../java/org/apache/doris/datasource/ExternalCatalog.java | 2 +-
.../java/org/apache/doris/datasource/ExternalDatabase.java | 2 +-
.../external_table_p0/lower_case/test_conflict_name.groovy | 10 ++++++++--
.../lower_case/test_lower_case_meta_include.groovy | 10 ++++++++--
.../lower_case/test_lower_case_meta_show_and_select.groovy | 10 ++++++++--
...wer_case_meta_with_lower_table_conf_show_and_select.groovy | 11 ++++++++---
.../external_table_p0/lower_case/test_lower_case_mtmv.groovy | 10 ++++++++--
.../lower_case/test_meta_cache_select_without_refresh.groovy | 10 ++++++++--
.../lower_case/test_meta_names_mapping.groovy | 10 ++++++++--
.../lower_case/test_timing_refresh_catalog.groovy | 10 ++++++++--
.../suites/external_table_p0/lower_case/upgrade/load.groovy | 8 ++++++--
11 files changed, 72 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 81a0eac2a68..05ad1c07cb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -723,7 +723,7 @@ public abstract class ExternalCatalog
public void replayInitCatalog(InitCatalogLog log) {
// If the remote name is missing during upgrade, all databases in the
Map will be reinitialized.
- if (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty()) {
+ if (log.getCreateCount() > 0 && (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty())) {
dbNameToId = Maps.newConcurrentMap();
idToDb = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index e7eb3949eae..cbacf563c32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -191,7 +191,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
// If the remote name is missing during upgrade, all tables in the Map
will be reinitialized.
- if (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty()) {
+ if (log.getCreateCount() > 0 && (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty())) {
tableNameToId = Maps.newConcurrentMap();
idToTbl = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
index 6187d47b645..43d6a68c1ad 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
@@ -18,12 +18,16 @@
suite("test_conflict_name", "p0,external,doris,meta_names_mapping") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_conflict_name_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_conflict_name; """
sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """
sql """create database if not exists internal.external_conflict_name; """
@@ -91,4 +95,6 @@ suite("test_conflict_name",
"p0,external,doris,meta_names_mapping") {
sql """drop database if exists internal.external_conflict_name; """
sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
index 854eb06a8e2..cd48678f25c 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_meta_include",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_include_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String mapping_db = """
{
"databases": [
@@ -155,4 +159,6 @@ suite("test_lower_case_meta_include",
"p0,external,doris,external_docker,externa
sql """drop catalog if exists test_lower_case_exclude """
sql """drop database if exists internal.external_INCLUDE; """
sql """drop database if exists internal.external_EXCLUDE; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
index d4efd141c2d..7b61216e949 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_without_conf_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_test_lower; """
sql """drop database if exists internal.external_test_UPPER; """
sql """create database if not exists internal.external_test_lower; """
@@ -251,4 +255,6 @@ suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker
sql """drop database if exists internal.external_test_lower; """
sql """drop database if exists internal.external_test_UPPER; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
index c3da24e1f55..d16648fc5e1 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
@@ -18,14 +18,17 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_with_conf"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_test_lower_with_conf; """
- sql """drop database if exists internal.external_test_UPPER_with_conf; """
sql """create database if not exists
internal.external_test_lower_with_conf; """
sql """create table if not exists
internal.external_test_lower_with_conf.lower_with_conf
(id int, name varchar(20))
@@ -699,4 +702,6 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """
sql """drop database if exists internal.external_test_lower_with_conf; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
index 40322a22afe..f72b3c9e68d 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_mtmv",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_case_mtmv_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """
sql """create database if not exists internal.EXTERNAL_LOWER_MTMV;"""
sql """create table if not exists internal.EXTERNAL_LOWER_MTMV.TABLE_TEST
@@ -61,4 +65,6 @@ suite("test_lower_case_mtmv",
"p0,external,doris,external_docker,external_docker
sql """drop catalog if exists test_lower_case_mtmv """
sql """drop database if exists internal.EXTERNAL_LOWER_MTMV """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
index 20e6b0f0032..17f37b2c132 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
@@ -18,12 +18,16 @@
suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_meta_cache_select_without_refresh_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """ drop database if exists
internal.external_lower_select_without_refresh; """
sql """create database if not exists
internal.external_lower_select_without_refresh;"""
@@ -89,4 +93,6 @@ suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_dock
sql """drop catalog if exists
test_meta_cache_lower_true_select_without_refresh """
sql """drop catalog if exists
test_meta_cache_lower_false_select_without_refresh """
sql """drop database if exists
internal.external_lower_select_without_refresh; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
index 1cf48b17c87..b58077295b3 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
@@ -18,12 +18,16 @@
suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_meta_names_mapping_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String validMetaNamesMapping = """
{
"databases": [
@@ -286,4 +290,6 @@ suite("test_meta_names_mapping",
"p0,external,doris,meta_names_mapping") {
sql """drop database if exists internal.external_meta_names_mapping; """
sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
index 19d7e2db180..a95da3b5d67 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
@@ -18,12 +18,16 @@
suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_timing_refresh_catalog_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String mapping = """
{
"databases": [
@@ -158,4 +162,6 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
sql """drop catalog if exists test_timing_refresh_catalog1 """
sql """drop catalog if exists test_timing_refresh_catalog2 """
sql """drop database if exists internal.external_timing_refresh_catalog """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
index 0ea89d4012d..ef03f97d0e2 100644
--- a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
@@ -18,12 +18,16 @@
suite("test_upgrade_lower_case_catalog_prepare",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_upgrade_lower_case_catalog_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.upgrade_lower_case_catalog_lower;
"""
sql """drop database if exists internal.upgrade_lower_case_catalog_UPPER;
"""
sql """create database if not exists
internal.upgrade_lower_case_catalog_lower; """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
