This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 47b2097dd025965e8dc6ab016d6b6728e6e8e2bc Author: Xiangyu Wang <[email protected]> AuthorDate: Thu May 11 01:00:55 2023 +0800 [Fix](multi-catalog) Fix sync hms event failed when start FE soon. (#19344) * [Fix](multi-catalog) Fix sync hms event failed when start FE soon after. * [Fix](multi-catalog) Fix sync hms event failed when start FE soon after. --------- Co-authored-by: [email protected] <[email protected]> --- .../org/apache/doris/catalog/RefreshManager.java | 2 +- .../org/apache/doris/datasource/CatalogMgr.java | 79 ++++++++++++++++------ .../datasource/hive/event/AddPartitionEvent.java | 2 +- .../datasource/hive/event/AlterPartitionEvent.java | 6 +- .../datasource/hive/event/AlterTableEvent.java | 10 +-- .../datasource/hive/event/CreateDatabaseEvent.java | 2 +- .../datasource/hive/event/CreateTableEvent.java | 11 +-- .../datasource/hive/event/DropDatabaseEvent.java | 2 +- .../datasource/hive/event/DropPartitionEvent.java | 2 +- .../datasource/hive/event/DropTableEvent.java | 2 +- .../doris/datasource/hive/event/InsertEvent.java | 2 +- 11 files changed, 77 insertions(+), 43 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index ef95b3e5f8..cf6a490b28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -56,7 +56,7 @@ public class RefreshManager { refreshInternalCtlIcebergTable(stmt, env); } else { // Process external catalog table refresh - env.getCatalogMgr().refreshExternalTable(dbName, tableName, catalogName); + env.getCatalogMgr().refreshExternalTable(dbName, tableName, catalogName, false); } LOG.info("Successfully refresh table: {} from db: {}", tableName, dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 1222f90432..df74d9a123 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -554,7 +554,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } } - public void refreshExternalTable(String dbName, String tableName, String catalogName) throws DdlException { + public void refreshExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfNotExists) + throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -564,12 +565,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + if (!ignoreIfNotExists) { + throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + } + return; } TableIf table = db.getTableNullable(tableName); if (table == null) { - throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + if (!ignoreIfNotExists) { + throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + } + return; } if (table instanceof ExternalTable) { ((ExternalTable) table).unsetObjectCreated(); @@ -603,7 +610,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { .invalidateTableCache(catalog.getId(), db.getFullName(), table.getName()); } - public void dropExternalTable(String dbName, String tableName, String catalogName) throws DdlException { + public void dropExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists) + throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -613,12 +621,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + if (!ignoreIfExists) { + throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + } + return; } TableIf table = db.getTableNullable(tableName); if (table == null) { - throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + if (!ignoreIfExists) { + throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + } + return; } ExternalObjectLog log = new ExternalObjectLog(); log.setCatalogId(catalog.getId()); @@ -668,7 +682,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { return ((ExternalCatalog) catalog).tableExistInLocal(dbName, tableName); } - public void createExternalTable(String dbName, String tableName, String catalogName) throws DdlException { + public void createExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists) + throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -678,12 +693,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + if (!ignoreIfExists) { + throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + } + return; } TableIf table = db.getTableNullable(tableName); if (table != null) { - throw new DdlException("Table " + tableName + " has exist in db " + dbName); + if (!ignoreIfExists) { + throw new DdlException("Table " + tableName + " has exist in db " + dbName); + } + return; } ExternalObjectLog log = new ExternalObjectLog(); log.setCatalogId(catalog.getId()); @@ -715,7 +736,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } } - public void dropExternalDatabase(String dbName, String catalogName) throws DdlException { + public void dropExternalDatabase(String dbName, String catalogName, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -725,7 +746,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + if (!ignoreIfNotExists) { + throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + } + return; } ExternalObjectLog log = new ExternalObjectLog(); @@ -758,7 +782,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } } - public void createExternalDatabase(String dbName, String catalogName) throws DdlException { + public void createExternalDatabase(String dbName, String catalogName, boolean ignoreIfExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -768,7 +792,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db != null) { - throw new DdlException("Database " + dbName + " has exist in catalog " + catalog.getName()); + if (!ignoreIfExists) { + throw new DdlException("Database " + dbName + " has exist in catalog " + catalog.getName()); + } + return; } ExternalObjectLog log = new ExternalObjectLog(); @@ -795,7 +822,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } } - public void addExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames) + public void addExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames, + boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { @@ -806,12 +834,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + if (!ignoreIfNotExists) { + throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + } + return; } TableIf table = db.getTableNullable(tableName); if (table == null) { - throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + if (!ignoreIfNotExists) { + throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + } + return; } ExternalObjectLog log = new ExternalObjectLog(); @@ -845,7 +879,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { .addPartitionsCache(catalog.getId(), table, log.getPartitionNames()); } - public void dropExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames) + public void dropExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames, + boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { @@ -856,12 +891,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + if (!ignoreIfNotExists) { + throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); + } + return; } TableIf table = db.getTableNullable(tableName); if (table == null) { - throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + if (!ignoreIfNotExists) { + throw new DdlException("Table " + tableName + " does not exist in db " + dbName); + } + return; } ExternalObjectLog log = new ExternalObjectLog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java index e5f8d1bb71..9ced60b322 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java @@ -79,7 +79,7 @@ public class AddPartitionEvent extends MetastoreTableEvent { return; } Env.getCurrentEnv().getCatalogMgr() - .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames); + .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java index 5209edccab..1e2eb6d06c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java @@ -79,9 +79,11 @@ public class AlterPartitionEvent extends MetastoreTableEvent { catalogName, dbName, tblName, partitionNameBefore, partitionNameAfter); if (isRename) { Env.getCurrentEnv().getCatalogMgr() - .dropExternalPartitions(catalogName, dbName, tblName, Lists.newArrayList(partitionNameBefore)); + .dropExternalPartitions(catalogName, dbName, tblName, + Lists.newArrayList(partitionNameBefore), true); Env.getCurrentEnv().getCatalogMgr() - .addExternalPartitions(catalogName, dbName, tblName, Lists.newArrayList(partitionNameAfter)); + .addExternalPartitions(catalogName, dbName, tblName, + Lists.newArrayList(partitionNameAfter), true); } else { Env.getCurrentEnv().getCatalogMgr() .refreshExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java index 9b731461cd..902ceacfa7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java @@ -73,9 +73,9 @@ public class AlterTableEvent extends MetastoreTableEvent { return; } Env.getCurrentEnv().getCatalogMgr() - .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName); + .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName, true); Env.getCurrentEnv().getCatalogMgr() - .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName); + .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName, true); } private void processRename() throws DdlException { @@ -91,9 +91,9 @@ public class AlterTableEvent extends MetastoreTableEvent { return; } Env.getCurrentEnv().getCatalogMgr() - .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName); + .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName, true); Env.getCurrentEnv().getCatalogMgr() - .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName); + .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName, true); } @@ -118,7 +118,7 @@ public class AlterTableEvent extends MetastoreTableEvent { } //The scope of refresh can be narrowed in the future Env.getCurrentEnv().getCatalogMgr() - .refreshExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName); + .refreshExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName, true); } catch (Exception e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java index 48476bce57..2205356caf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java @@ -48,7 +48,7 @@ public class CreateDatabaseEvent extends MetastoreEvent { try { infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName); Env.getCurrentEnv().getCatalogMgr() - .createExternalDatabase(dbName, catalogName); + .createExternalDatabase(dbName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java index 2d364c4c14..1dbfd08ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java @@ -59,16 +59,7 @@ public class CreateTableEvent extends MetastoreTableEvent { protected void process() throws MetastoreNotificationException { try { infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tblName); - boolean hasExist = Env.getCurrentEnv().getCatalogMgr() - .externalTableExistInLocal(dbName, hmsTbl.getTableName(), catalogName); - if (hasExist) { - infoLog( - "CreateExternalTable canceled,because table has exist," - + "catalogName:[{}],dbName:[{}],tableName:[{}]", - catalogName, dbName, tblName); - return; - } - Env.getCurrentEnv().getCatalogMgr().createExternalTable(dbName, hmsTbl.getTableName(), catalogName); + Env.getCurrentEnv().getCatalogMgr().createExternalTable(dbName, hmsTbl.getTableName(), catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java index b51145a258..a11e893ef2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java @@ -48,7 +48,7 @@ public class DropDatabaseEvent extends MetastoreEvent { try { infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName); Env.getCurrentEnv().getCatalogMgr() - .dropExternalDatabase(dbName, catalogName); + .dropExternalDatabase(dbName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java index 1dce403f55..59254fc4f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java @@ -79,7 +79,7 @@ public class DropPartitionEvent extends MetastoreTableEvent { return; } Env.getCurrentEnv().getCatalogMgr() - .dropExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames); + .dropExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java index aa74c67512..d42de68cbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java @@ -59,7 +59,7 @@ public class DropTableEvent extends MetastoreTableEvent { protected void process() throws MetastoreNotificationException { try { infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tableName); - Env.getCurrentEnv().getCatalogMgr().dropExternalTable(dbName, tableName, catalogName); + Env.getCurrentEnv().getCatalogMgr().dropExternalTable(dbName, tableName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java index cf4ba1d5b0..2b53e86639 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java @@ -66,7 +66,7 @@ public class InsertEvent extends MetastoreTableEvent { * the file cache of this table, * but <a href="https://github.com/apache/doris/pull/17932";>this PR</a> has fixed it. */ - Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(dbName, tblName, catalogName); + Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(dbName, tblName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event")); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
