924060929 commented on code in PR #68196:
URL: https://github.com/apache/doris/pull/68196#discussion_r4132374299
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java:
##########
@@ -55,7 +55,8 @@ protected static List<MetastoreEvent>
getEvents(NotificationEvent event,
protected void process() throws MetastoreNotificationException {
try {
logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName);
-
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName,
catalogName);
+
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(
+ event == null ? dbName : event.getDbName(), catalogName);
Review Comment:
Fixed in 0c37f5145e4. HMS CREATE now derives the local database name and
deterministic ID through the same mode-1 normalization as discovery; the
original name remains the remote identity. CatalogMgrTest covers CREATE and
rename registration followed by DROP. I also aligned mode-1/2 include/exclude
filtering with normalized HMS event names; the previously failing
ExternalCatalogDeadlockTest now passes.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java:
##########
@@ -520,9 +597,51 @@ private void invalidateLanceTableAccess(long catalogId) {
public void invalidatePartitions(long catalogId,
String dbName, String tableName, List<String> partitions) {
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "invalidatePartitions",
- () -> cache.invalidatePartitions(catalogId, dbName, tableName,
partitions)));
+ Runnable rowCountFence = () ->
rowCountCache.invalidateCatalog(catalogId);
+ try {
+ rowCountFence = resolveTableRowCountFence(catalogId, dbName,
getCachedDb(catalogId, dbName), tableName);
+ rowCountFence.run();
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "invalidatePartitions",
+ () -> cache.invalidatePartitions(catalogId, dbName,
tableName, partitions)));
+ } finally {
+ rowCountFence.run();
+ }
+ }
+
+ private Runnable resolveTableRowCountFence(long catalogId, String dbName,
+ Optional<ExternalDatabase<? extends ExternalTable>> db, String
tableName) {
+ if (db.isPresent()) {
+ Optional<? extends ExternalTable> table =
db.get().getTableForReplay(tableName);
+ if (table.isPresent()) {
+ long tableCatalogId = table.get().getCatalog().getId();
+ long tableDbId = table.get().getDb().getId();
+ long tableId = table.get().getId();
+ return () -> rowCountCache.invalidateTable(tableCatalogId,
tableDbId, tableId);
+ }
+ long dbId = db.get().getId();
Review Comment:
Fixed in 0c37f5145e4. The cold-table path resolves the retained canonical
local name to its deterministic table ID and fences that exact row-count key.
It widens to DB only when the name mapping is genuinely unavailable.
ExternalMetaCacheRouteResolverTest covers both cases.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java:
##########
@@ -332,8 +426,15 @@ public void refreshPartitions(String catalogName, String
dbName, String tableNam
ExternalTable externalTable = (ExternalTable) table;
HiveExternalMetaCache cache =
Env.getCurrentEnv().getExtMetaCacheMgr().hive(externalTable.getCatalog().getId());
- for (String partitionName : partitionNames) {
- cache.invalidatePartitionCache(externalTable, partitionName);
+ try {
Review Comment:
Fixed in 0c37f5145e4. ADD, DROP, and ALTER PARTITION now wrap the entire
post-opening-fence path, including table/schema lookup and Hive cache
acquisition, in a closing-fence finally block. Acquisition-failure and
early-return tests require both fences.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java:
##########
@@ -120,16 +120,25 @@ public void setRemoteName(String remoteName) {
}
public void resetMetaToUninitialized() {
- resetMetaToUninitialized(true);
+ resetMetaToUninitialized(true, true);
}
public void resetMetaToUninitialized(boolean invalidateEngineCache) {
+ resetMetaToUninitialized(invalidateEngineCache, invalidateEngineCache);
+ }
+
+ public void resetMetaToUninitialized(boolean invalidateEngineCache,
boolean invalidateRowCountCache) {
if (LOG.isDebugEnabled()) {
LOG.debug("resetToUninitialized db name {}, id {}, isInitializing:
{}, initialized: {}",
this.name, this.id, isInitializing, initialized, new
Exception());
}
MetaCache<T> cacheToInvalidate = null;
Runnable objectInvalidation = null;
+ if (invalidateRowCountCache) {
Review Comment:
Fixed in 0c37f5145e4. Leader REFRESH DATABASE and replay REFRESH
DATABASE/TABLE now fence the DB or table row-count identity before retiring
Lance access; existing completion fences remain. Lance lifecycle and
RefreshManager tests assert the ordering.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java:
##########
@@ -714,26 +711,42 @@ public void resetToUninitialized(boolean invalidCache) {
*/
public void onRefreshCache(boolean invalidCache) {
setLastUpdateTime(System.currentTimeMillis());
- refreshMetaCacheOnly(invalidCache);
- if (invalidCache) {
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id);
+ try {
+ refreshMetaCacheOnly(invalidCache);
+ } finally {
+ if (invalidCache) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id);
+ }
}
}
/**
* Refresh meta cache only (database level cache), without invalidating
catalog level cache.
*/
private void refreshMetaCacheOnly(boolean invalidCache) {
- if (metaCache != null) {
- // A catalog-wide engine invalidation below supersedes every
database invalidation.
- // The legacy cache uses a synchronous removal listener, so this
thread-local scope
- // prevents one full SDK-cache scan per cached database without
affecting concurrent
- // expiry callbacks on other threads.
- invalidateEngineCacheOnDatabaseRemoval.set(!invalidCache);
- try {
- metaCache.invalidateAll();
- } finally {
- invalidateEngineCacheOnDatabaseRemoval.remove();
+ Runnable objectInvalidation;
+ ExternalMetaCacheMgr cacheMgr =
Env.getCurrentEnv().getExtMetaCacheMgr();
+ synchronized (this) {
+ if (metaCache == null) {
+ return;
+ }
+ // Publish the names/object generation transition together, before
allowing another
+ // catalog initialization. The old cache removal callbacks can be
much slower.
+ cacheMgr.invalidateRowCountCache(id);
Review Comment:
Fixed in 0c37f5145e4. Catalog row-count invalidation now advances a globally
unique per-catalog generation in O(1) under the publication lock, so the
catalog monitor no longer performs the 100,000-entry sweep. Old cache entries
are bounded by Caffeine size/expiry and cannot be addressed by a new
generation; an in-flight old read checks generation again before returning.
ExternalRowCountCacheTest covers the refresh/invalidation race.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
