This is an automated email from the ASF dual-hosted git repository.
zykkk pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new fc76577c24b branch-3.1: [fix](catalog) Prevent removing catalog from
refresh queue on CREATE/ALTER (#57671)
fc76577c24b is described below
commit fc76577c24bd28f978613e1aab658c9b56c05e95
Author: zy-kkk <[email protected]>
AuthorDate: Wed Nov 5 17:54:02 2025 +0800
branch-3.1: [fix](catalog) Prevent removing catalog from refresh queue on
CREATE/ALTER (#57671)
Problem Summary:
catalogs are incorrectly removed from the auto-refresh queue when:
- Creating a new catalog with `metadata_refresh_interval_sec`
- Altering catalog properties via `ALTER CATALOG SET PROPERTIES`
This causes auto-refresh to stop working even though
`metadata_refresh_interval_sec` is configured.
Root Cause:
CREATE/ALTER CATALOG
→ resetToUninitialized()
→ onClose()
→ CatalogIf.super.onClose()
→ removeFromRefreshMap(catalogId) ❌ Wrong!
Fix:
1. Remove `CatalogIf.super.onClose()` call from
`ExternalCatalog.onClose()`
- Prevents wrongly removing catalog from refresh queue during
CREATE/ALTER
2. Explicitly call `removeFromRefreshMap()` in
`CatalogMgr.removeCatalog()`
- Ensures catalog is properly removed when actually dropping it
---
.../org/apache/doris/datasource/CatalogMgr.java | 2 +
.../apache/doris/datasource/ExternalCatalog.java | 1 -
.../doris/datasource/RefreshCatalogTest.java | 2 +-
.../test_hive_metadata_refresh_interval.groovy | 101 +++++++++++++++++++++
4 files changed, 104 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 1564c89ce53..57029ec5cc1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -131,6 +131,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
CatalogIf catalog = idToCatalog.remove(catalogId);
LOG.info("Removed catalog with id {}, name {}", catalogId, catalog ==
null ? "N/A" : catalog.getName());
if (catalog != null) {
+ // Remove from refresh map before calling onClose()
+
Env.getCurrentEnv().getRefreshManager().removeFromRefreshMap(catalogId);
catalog.onClose();
nameToCatalog.remove(catalog.getName());
if (ConnectContext.get() != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 0677a12050a..a0a5b9a42de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -815,7 +815,6 @@ public abstract class ExternalCatalog
if (threadPoolWithPreAuth != null) {
ThreadPoolManager.shutdownExecutorService(threadPoolWithPreAuth);
}
- CatalogIf.super.onClose();
}
private void removeAccessController() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 4e80a971f1a..56068c210fe 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -102,7 +102,7 @@ public class RefreshCatalogTest extends TestWithFeService {
Thread.sleep(5000);
// there are test1.db1 , test1.db2 , test1.db3, information_schema,
mysql
List<String> dbNames2 = test1.getDbNames();
- Assertions.assertEquals(4, dbNames2.size());
+ Assertions.assertEquals(5, dbNames2.size());
ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase)
test1.getDb(InfoSchemaDb.DATABASE_NAME).get();
Assertions.assertEquals(SchemaTable.TABLE_MAP.size(),
infoDb.getTables().size());
TestExternalDatabase testDb = (TestExternalDatabase)
test1.getDb("db1").get();
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_metadata_refresh_interval.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_metadata_refresh_interval.groovy
new file mode 100644
index 00000000000..1b5c6d5dc70
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_metadata_refresh_interval.groovy
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_metadata_refresh_interval",
"p0,external,hive,external_docker,external_docker_hive") {
+
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Hive test.")
+ return
+ }
+
+ for (String hivePrefix : ["hive2", "hive3"]) {
+ setHivePrefix(hivePrefix)
+ try {
+ String hms_port = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String hdfs_port = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+ String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+
+ String catalog_name = "test_${hivePrefix}_refresh_interval"
+ String test_db = "test_refresh_interval_db"
+ String table1 = "test_refresh_table_1"
+ String table2 = "test_refresh_table_2"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ hive_docker "drop database if exists ${test_db} cascade"
+ hive_docker "create database ${test_db}"
+
+ // Create catalog with 10s refresh interval
+ sql """create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hms_port}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'use_meta_cache' = 'true',
+ 'metadata_refresh_interval_sec' = '10'
+ )"""
+
+ sql "switch ${catalog_name}"
+ sql "use ${test_db}"
+
+ // Show tables - should be empty
+ def result1 = sql "show tables from ${test_db}"
+ assertEquals(0, result1.size())
+
+ // Create table1 via hive_docker
+ hive_docker "create table ${test_db}.${table1} (id int, name
string)"
+
+ // Wait 10s for automatic refresh
+ sleep(10000)
+
+ // Show tables - should have table1
+ def result2 = sql "show tables from ${test_db}"
+ assertEquals(1, result2.size())
+ assertTrue(result2.toString().contains(table1))
+
+ // ALTER catalog to 60s refresh interval
+ sql """alter catalog ${catalog_name} set properties (
+ 'metadata_refresh_interval_sec' = '60'
+ )"""
+
+ // Show tables - should still have only table1
+ def result3 = sql "show tables from ${test_db}"
+ assertEquals(1, result3.size())
+ assertTrue(result3.toString().contains(table1))
+ assertFalse(result3.toString().contains(table2))
+
+ // Create table2 via hive_docker
+ hive_docker "create table ${test_db}.${table2} (id int, value
string)"
+
+ // Wait 60s for automatic refresh
+ sleep(60000)
+
+ // Show tables - should have both table1 and table2
+ def result4 = sql "show tables from ${test_db}"
+ assertEquals(2, result4.size())
+ assertTrue(result4.toString().contains(table1))
+ assertTrue(result4.toString().contains(table2))
+
+ // Cleanup
+ hive_docker "drop database ${test_db} cascade"
+ sql """drop catalog ${catalog_name}"""
+
+ } finally {
+ sql """drop catalog if exists
test_${hivePrefix}_refresh_interval"""
+ hive_docker "drop database if exists test_refresh_interval_db
cascade"
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
