This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 403fa358251 branch-4.2: [fix](fe) Avoid database object loading for
schemata scans #70010 (#70104)
403fa358251 is described below
commit 403fa3582519d16c38d2c304016acf9806e42d6d
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Nov 4 08:29:36 2026 +0800
branch-4.2: [fix](fe) Avoid database object loading for schemata scans
#70010 (#70104)
Cherry-picked from #70010
Co-authored-by: Gabriel <[email protected]>
---
.../information_schema/schema_schemata_scanner.cpp | 2 +
.../apache/doris/service/FrontendServiceImpl.java | 54 ++++--
.../service/FrontendServiceGetDbNamesTest.java | 186 +++++++++++++++++++++
gensrc/thrift/FrontendService.thrift | 2 +
4 files changed, 229 insertions(+), 15 deletions(-)
diff --git a/be/src/information_schema/schema_schemata_scanner.cpp
b/be/src/information_schema/schema_schemata_scanner.cpp
index e9d25684780..15e2528a89f 100644
--- a/be/src/information_schema/schema_schemata_scanner.cpp
+++ b/be/src/information_schema/schema_schemata_scanner.cpp
@@ -52,6 +52,8 @@ Status SchemaSchemataScanner::start(RuntimeState* state) {
return Status::InternalError("used before initial.");
}
TGetDbsParams db_params;
+ // SCHEMATA never reads database IDs; avoid creating one database object
per namespace in FE.
+ db_params.__set_include_db_ids(false);
if (nullptr != _param->common_param->wild) {
db_params.__set_pattern(*(_param->common_param->wild));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 23c3a74f1d4..30493552832 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -345,7 +345,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -363,6 +362,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.ObjLongConsumer;
import java.util.stream.Collectors;
// Frontend service used to serve all request for this frontend through
@@ -496,7 +496,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
List<Long> dbIds = Lists.newArrayList();
List<Long> catalogIds = Lists.newArrayList();
- PatternMatcher matcher = null;
+ PatternMatcher matcher;
if (params.isSetPattern()) {
try {
matcher =
PatternMatcher.createMysqlPattern(params.getPattern(),
@@ -504,6 +504,8 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
} catch (PatternMatcherException e) {
throw new TException("Pattern is in bad format: " +
params.getPattern());
}
+ } else {
+ matcher = null;
}
Env env = Env.getCurrentEnv();
@@ -517,55 +519,77 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
catalog -> new TException("Unknown catalog " +
catalog)));
}
+ boolean includeDbIds = params.isIncludeDbIds();
for (CatalogIf catalog : catalogIfs) {
- Collection<DatabaseIf> dbs = new HashSet<DatabaseIf>();
+ Collection<DatabaseIf> dbs = Collections.emptyList();
+ List<String> names = Collections.emptyList();
try {
- dbs = catalog.getAllDbs();
+ if (includeDbIds) {
+ dbs = catalog.getAllDbs();
+ } else {
+ // SCHEMATA only needs names; loading every database
churns the object cache on large catalogs.
+ names = catalog.getDbNames();
+ }
} catch (Exception e) {
LOG.warn("failed to get database names for catalog {}",
catalog.getName(), e);
// Some external catalog may fail to get databases due to
wrong connection info.
}
+ int dbCount = includeDbIds ? dbs.size() : names.size();
if (LOG.isDebugEnabled()) {
- LOG.debug("get db size: {}, in catalog: {}", dbs.size(),
catalog.getName());
+ LOG.debug("get db size: {}, in catalog: {}", dbCount,
catalog.getName());
}
- if (dbs.isEmpty() && params.isSetGetNullCatalog() &&
params.get_null_catalog) {
+ if (dbCount == 0 && params.isSetGetNullCatalog() &&
params.get_null_catalog) {
catalogNames.add(catalog.getName());
dbNames.add("NULL");
catalogIds.add(catalog.getId());
- dbIds.add(-1L);
+ if (includeDbIds) {
+ dbIds.add(-1L);
+ }
continue;
}
- if (dbs.isEmpty()) {
+ if (dbCount == 0) {
continue;
}
- UserIdentity currentUser = null;
+ UserIdentity currentUser;
if (params.isSetCurrentUserIdent()) {
currentUser =
UserIdentity.fromThrift(params.current_user_ident);
} else {
currentUser =
UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip);
}
- for (DatabaseIf db : dbs) {
- String dbName = db.getFullName();
+ ObjLongConsumer<String> addDatabase = (dbName, dbId) -> {
if (!env.getAccessManager()
.checkDbPriv(currentUser, catalog.getName(), dbName,
PrivPredicate.SHOW)) {
- continue;
+ return;
}
if (matcher != null &&
!matcher.match(getMysqlTableSchema(catalog.getName(), dbName))) {
- continue;
+ return;
}
catalogNames.add(catalog.getName());
dbNames.add(getMysqlTableSchema(catalog.getName(), dbName));
catalogIds.add(catalog.getId());
- dbIds.add(db.getId());
+ if (includeDbIds) {
+ dbIds.add(dbId);
+ }
+ };
+ if (includeDbIds) {
+ for (DatabaseIf db : dbs) {
+ addDatabase.accept(db.getFullName(), db.getId());
+ }
+ } else {
+ for (String dbName : names) {
+ addDatabase.accept(dbName, -1L);
+ }
}
}
result.setDbs(dbNames);
result.setCatalogs(catalogNames);
result.setCatalogIds(catalogIds);
- result.setDbIds(dbIds);
+ if (includeDbIds) {
+ result.setDbIds(dbIds);
+ }
return result;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceGetDbNamesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceGetDbNamesTest.java
new file mode 100644
index 00000000000..236569257fd
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceGetDbNamesTest.java
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.InfoSchemaDb;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.CatalogMgr;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.GlobalVariable;
+import org.apache.doris.thrift.TGetDbsParams;
+import org.apache.doris.thrift.TGetDbsResult;
+
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class FrontendServiceGetDbNamesTest {
+ private final Env env = Mockito.mock(Env.class);
+ private final CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ private final AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ private final CatalogIf catalog = Mockito.mock(CatalogIf.class);
+ private final FrontendServiceImpl service = new
FrontendServiceImpl(Mockito.mock(ExecuteEnv.class));
+ private MockedStatic<Env> envMock;
+
+ @BeforeEach
+ public void setUp() {
+ envMock = Mockito.mockStatic(Env.class);
+ envMock.when(Env::getCurrentEnv).thenReturn(env);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+
Mockito.when(catalogMgr.listCatalogs()).thenReturn(Collections.singletonList(catalog));
+ Mockito.when(catalog.getName()).thenReturn("example_catalog");
+ Mockito.when(catalog.getId()).thenReturn(42L);
+
Mockito.when(accessManager.checkDbPriv(Mockito.any(UserIdentity.class),
Mockito.anyString(),
+ Mockito.anyString(),
Mockito.eq(PrivPredicate.SHOW))).thenReturn(true);
+ }
+
+ @AfterEach
+ public void tearDown() {
+ envMock.close();
+ }
+
+ @Test
+ public void testNamesOnlyDoesNotMaterializeDatabases() throws Exception {
+ List<String> names = new ArrayList<>();
+ for (int i = 0; i < 100_000; i++) {
+ names.add("namespace_" + i);
+ }
+ names.add(InfoSchemaDb.DATABASE_NAME);
+ names.add("mysql");
+ Mockito.when(catalog.getDbNames()).thenReturn(names);
+ Mockito.when(catalog.getAllDbs()).thenThrow(new
AssertionError("Database objects must not be loaded"));
+
+ TGetDbsParams params = params();
+ params.setIncludeDbIds(false);
+ TGetDbsResult result = service.getDbNames(params);
+
+ Assertions.assertEquals(names.size(), result.getDbsSize());
+ Assertions.assertEquals(names.size(), result.getCatalogsSize());
+ Assertions.assertEquals(names.size(), result.getCatalogIdsSize());
+
Assertions.assertTrue(result.getCatalogs().stream().allMatch("example_catalog"::equals));
+ Assertions.assertTrue(result.getCatalogIds().stream().allMatch(id ->
id == 42L));
+ Assertions.assertFalse(result.isSetDbIds());
+ Mockito.verify(catalog, Mockito.times(1)).getDbNames();
+ Mockito.verify(catalog, Mockito.never()).getAllDbs();
+ }
+
+ @Test
+ public void testNamesOnlyPreservesPrivilegesPatternAndDisplayNames()
throws Exception {
+
Mockito.when(catalog.getDbNames()).thenReturn(Arrays.asList("visible_a",
"visible_hidden", "other"));
+
Mockito.when(accessManager.checkDbPriv(Mockito.any(UserIdentity.class),
Mockito.eq("example_catalog"),
+ Mockito.eq("visible_hidden"),
Mockito.eq(PrivPredicate.SHOW))).thenReturn(false);
+
+ boolean original = GlobalVariable.showFullDbNameInInfoSchemaDb;
+ try {
+ for (boolean showFullName : new boolean[] {false, true}) {
+ GlobalVariable.showFullDbNameInInfoSchemaDb = showFullName;
+ TGetDbsParams params = params();
+ params.setIncludeDbIds(false);
+ params.setPattern(showFullName ? "example_catalog.visible%" :
"visible%");
+ TGetDbsResult result = service.getDbNames(params);
+ Assertions.assertEquals(Collections.singletonList(
+ showFullName ? "example_catalog.visible_a" :
"visible_a"), result.getDbs());
+ Assertions.assertFalse(result.isSetDbIds());
+ }
+ } finally {
+ GlobalVariable.showFullDbNameInInfoSchemaDb = original;
+ }
+ }
+
+ @Test
+ public void testExistingCallersKeepDatabaseIds() throws Exception {
+ DatabaseIf database = Mockito.mock(DatabaseIf.class);
+ Mockito.when(database.getFullName()).thenReturn("visible_a");
+ Mockito.when(database.getId()).thenReturn(123L);
+
Mockito.when(catalog.getAllDbs()).thenReturn(Collections.singletonList(database));
+
+ for (boolean explicitlySet : new boolean[] {false, true}) {
+ TGetDbsParams params = params();
+ if (explicitlySet) {
+ params.setIncludeDbIds(true);
+ }
+ TGetDbsResult result = service.getDbNames(params);
+ Assertions.assertEquals(Collections.singletonList(123L),
result.getDbIds());
+ Assertions.assertEquals(Collections.singletonList(42L),
result.getCatalogIds());
+ }
+ Mockito.verify(catalog, Mockito.never()).getDbNames();
+ }
+
+ @Test
+ public void testAbsentIncludeDbIdsOnWireKeepsLegacyDefault() throws
Exception {
+ TGetDbsParams legacy = params();
+ legacy.unsetIncludeDbIds();
+ TSerializer serializer = new TSerializer(new
TBinaryProtocol.Factory());
+ TDeserializer deserializer = new TDeserializer(new
TBinaryProtocol.Factory());
+ TGetDbsParams decoded = new TGetDbsParams();
+ deserializer.deserialize(decoded, serializer.serialize(legacy));
+ Assertions.assertTrue(decoded.isIncludeDbIds());
+
+ legacy.setIncludeDbIds(false);
+ deserializer.deserialize(decoded, serializer.serialize(legacy));
+ Assertions.assertFalse(decoded.isIncludeDbIds());
+ }
+
+ @Test
+ public void testNamesOnlyPreservesEmptyCatalogPlaceholder() throws
Exception {
+ Mockito.when(catalog.getDbNames()).thenReturn(Collections.emptyList());
+ TGetDbsParams params = params();
+ params.setIncludeDbIds(false);
+ params.setGetNullCatalog(true);
+
+ TGetDbsResult result = service.getDbNames(params);
+ Assertions.assertEquals(Collections.singletonList("NULL"),
result.getDbs());
+ Assertions.assertEquals(Collections.singletonList("example_catalog"),
result.getCatalogs());
+ Assertions.assertEquals(Collections.singletonList(42L),
result.getCatalogIds());
+ Assertions.assertFalse(result.isSetDbIds());
+ }
+
+ @Test
+ public void testNamesOnlySkipsUnavailableCatalog() throws Exception {
+ Mockito.when(catalog.getDbNames()).thenThrow(new
IllegalStateException("Metadata service unavailable"));
+ TGetDbsParams params = params();
+ params.setIncludeDbIds(false);
+
+ TGetDbsResult result = service.getDbNames(params);
+ Assertions.assertTrue(result.getDbs().isEmpty());
+ Assertions.assertTrue(result.getCatalogs().isEmpty());
+ Assertions.assertFalse(result.isSetDbIds());
+ }
+
+ private TGetDbsParams params() {
+ TGetDbsParams params = new TGetDbsParams();
+ params.setCurrentUserIdent(UserIdentity.ROOT.toThrift());
+ return params;
+ }
+}
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index 45e63623fa8..094a52d22ef 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -101,6 +101,8 @@ struct TGetDbsParams {
// Reserved for downstream field `current_roles` to keep thrift field ids
// wire-compatible across maintained branches. Do not reuse this id.
7: optional set<string> reserved_field_7
+ // Name-only scanners can avoid materializing database objects and omit
db_ids in the response.
+ 8: optional bool include_db_ids = true
}
// getDbNames returns a list of database names , database ids and catalog
names ,catalog ids
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
