This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 72caa403024 [opt](catalog) cache the converted properties (#30668)
72caa403024 is described below
commit 72caa403024e613bc84cbdd139ba801018ea662f
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon Feb 5 22:43:33 2024 +0800
[opt](catalog) cache the converted properties (#30668)
convert properties may be a heavy operation, so we cache the result.
---
.../org/apache/doris/datasource/ExternalCatalog.java | 20 +++++++++++++++++++-
.../datasource/jdbc/JdbcExternalCatalogTest.java | 1 +
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 6fcd495b67f..c5408dd3103 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -104,6 +104,10 @@ public abstract class ExternalCatalog
private ExternalSchemaCache schemaCache;
private String comment;
+ // A cached and being converted properties for external catalog.
+ // generated from catalog properties.
+ private byte[] propLock = new byte[0];
+ private Map<String, String> convertedProperties = null;
public ExternalCatalog() {
}
@@ -298,6 +302,9 @@ public abstract class ExternalCatalog
public void onRefresh(boolean invalidCache) {
this.objectCreated = false;
this.initialized = false;
+ synchronized (this.propLock) {
+ this.convertedProperties = null;
+ }
this.invalidCacheInInit = invalidCache;
if (invalidCache) {
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalogCache(id);
@@ -421,7 +428,17 @@ public abstract class ExternalCatalog
@Override
public Map<String, String> getProperties() {
- return
PropertyConverter.convertToMetaProperties(catalogProperty.getProperties());
+ // convert properties may be a heavy operation, so we cache the result.
+ if (convertedProperties != null) {
+ return convertedProperties;
+ }
+ synchronized (propLock) {
+ if (convertedProperties != null) {
+ return convertedProperties;
+ }
+ convertedProperties =
PropertyConverter.convertToMetaProperties(catalogProperty.getProperties());
+ return convertedProperties;
+ }
}
@Override
@@ -552,6 +569,7 @@ public abstract class ExternalCatalog
}
}
}
+ this.propLock = new byte[0];
}
public void addDatabaseForTest(ExternalDatabase<? extends ExternalTable>
db) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
index 7bc268b6421..8394daf0682 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
@@ -54,6 +54,7 @@ public class JdbcExternalCatalogTest {
Map<String, String> newProperties = Maps.newHashMap();
newProperties.put(JdbcResource.CONNECTION_POOL_MIN_SIZE, "2");
jdbcExternalCatalog.getCatalogProperty().modifyCatalogProps(newProperties);
+ jdbcExternalCatalog.notifyPropertiesUpdated(newProperties);
JdbcExternalCatalog replayJdbcCatalog2 = (JdbcExternalCatalog)
CatalogFactory.createFromLog(
jdbcExternalCatalog.constructEditLog());
Map<String, String> properties2 = replayJdbcCatalog2.getProperties();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
