This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 1d997c08105 branch-3.1: [opt](iceberg)Improve performance by not
retrieving table objects for hms (#47782) (#51942)
1d997c08105 is described below
commit 1d997c0810506e4ddc8ffc86cac3df3fa58ac3ef
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu Jun 19 19:35:14 2025 +0800
branch-3.1: [opt](iceberg)Improve performance by not retrieving table
objects for hms (#47782) (#51942)
bp #47782
Co-authored-by: wuwenchi <[email protected]>
---
.../doris/datasource/iceberg/IcebergUtils.java | 6 +++
.../doris/datasource/iceberg/IcebergUtilsTest.java | 63 ++++++++++++++++++++++
2 files changed, 69 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index 76e9781b78b..5ef64153418 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -683,6 +683,12 @@ public class IcebergUtils {
hiveCatalog.setConf(externalCatalog.getConfiguration());
Map<String, String> catalogProperties =
externalCatalog.getProperties();
+ if (!catalogProperties.containsKey(HiveCatalog.LIST_ALL_TABLES)) {
+ // This configuration will display all tables (including
non-Iceberg type tables),
+ // which can save the time of obtaining table objects.
+ // Later, type checks will be performed when loading the table.
+ catalogProperties.put(HiveCatalog.LIST_ALL_TABLES, "true");
+ }
String metastoreUris =
catalogProperties.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, "");
catalogProperties.put(CatalogProperties.URI, metastoreUris);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
new file mode 100644
index 00000000000..244f173acb7
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.iceberg.hive.HiveCatalog;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+
+public class IcebergUtilsTest {
+ @Test
+ public void testParseTableName() {
+ try {
+ IcebergHMSExternalCatalog c1 =
+ new IcebergHMSExternalCatalog(1, "name", null, new
HashMap<>(), "");
+ HiveCatalog i1 = IcebergUtils.createIcebergHiveCatalog(c1, "i1");
+ Assert.assertTrue(getListAllTables(i1));
+
+ IcebergHMSExternalCatalog c2 =
+ new IcebergHMSExternalCatalog(1, "name", null,
+ new HashMap<String, String>() {{
+ put("list-all-tables", "true");
+ }},
+ "");
+ HiveCatalog i2 = IcebergUtils.createIcebergHiveCatalog(c2, "i1");
+ Assert.assertTrue(getListAllTables(i2));
+
+ IcebergHMSExternalCatalog c3 =
+ new IcebergHMSExternalCatalog(1, "name", null,
+ new HashMap<String, String>() {{
+ put("list-all-tables", "false");
+ }},
+ "");
+ HiveCatalog i3 = IcebergUtils.createIcebergHiveCatalog(c3, "i1");
+ Assert.assertFalse(getListAllTables(i3));
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ private boolean getListAllTables(HiveCatalog hiveCatalog) throws
IllegalAccessException, NoSuchFieldException {
+ Field declaredField =
hiveCatalog.getClass().getDeclaredField("listAllTables");
+ declaredField.setAccessible(true);
+ return declaredField.getBoolean(hiveCatalog);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
