This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6764feeffeb [fix](hive) sync DDL command to other FE (#46326)
6764feeffeb is described below
commit 6764feeffebc661dfd50332c6756f35e76f9d593
Author: Mingyu Chen (Rayner) <morning...@163.com>
AuthorDate: Sun Feb 2 22:39:36 2025 +0800
[fix](hive) sync DDL command to other FE (#46326)
### What problem does this PR solve?
When executing DDL on a hive table, the DDL command should be synced to
other FE,
so that other FE can invalidate the cache of that table or db.
Otherwise, after executing the DDL on a hive table on non-master FE and
then query that table,
we can still access to the cached files.
Modify the following ddl:
1. create db/table
2. drop db/table
3. truncate table
For these operations, I saved the info like dbname, tablename, and sync
them to other FE,
so that other FE can refresh these objects to get latest metadata info
### Release Note
This PR add a new edit log type: `OperationType.OP_NEW_CREATE_DB`.
So that it can't downgrade
---
.../main/java/org/apache/doris/catalog/Env.java | 49 ++++++++++++---
.../org/apache/doris/datasource/CatalogIf.java | 4 +-
.../apache/doris/datasource/ExternalCatalog.java | 63 ++++++++++++++++---
.../apache/doris/datasource/InternalCatalog.java | 10 ++-
.../doris/datasource/hive/HiveMetadataOps.java | 53 +++++++++++-----
.../datasource/iceberg/IcebergMetadataOps.java | 45 ++++++++-----
.../datasource/operations/ExternalMetadataOps.java | 57 +++++++++++++++--
.../org/apache/doris/journal/JournalEntity.java | 6 ++
.../org/apache/doris/persist/CreateDbInfo.java | 73 ++++++++++++++++++++++
.../org/apache/doris/persist/CreateTableInfo.java | 30 ++++++++-
.../java/org/apache/doris/persist/DropDbInfo.java | 13 ++++
.../java/org/apache/doris/persist/DropInfo.java | 33 ++++++++++
.../java/org/apache/doris/persist/EditLog.java | 70 ++++++++++++++-------
.../org/apache/doris/persist/OperationType.java | 4 +-
.../apache/doris/persist/TruncateTableInfo.java | 27 +++++++-
.../org/apache/doris/persist/CreateDbInfoTest.java | 72 +++++++++++++++++++++
16 files changed, 525 insertions(+), 84 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 548c427a9bf..b1fc9ced5a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -211,6 +211,9 @@ import org.apache.doris.persist.BackendReplicasInfo;
import org.apache.doris.persist.BackendTabletsInfo;
import org.apache.doris.persist.BinlogGcInfo;
import org.apache.doris.persist.CleanQueryStatsInfo;
+import org.apache.doris.persist.CreateDbInfo;
+import org.apache.doris.persist.CreateTableInfo;
+import org.apache.doris.persist.DropDbInfo;
import org.apache.doris.persist.DropPartitionInfo;
import org.apache.doris.persist.EditLog;
import org.apache.doris.persist.GlobalVarPersistInfo;
@@ -3259,8 +3262,15 @@ public class Env {
getInternalCatalog().unprotectCreateDb(db);
}
- public void replayCreateDb(Database db) {
- getInternalCatalog().replayCreateDb(db, "");
+ public void replayCreateDb(CreateDbInfo dbInfo) {
+ if (dbInfo.getInternalDb() != null) {
+ getInternalCatalog().replayCreateDb(dbInfo.getInternalDb(), "");
+ } else {
+ ExternalCatalog externalCatalog = (ExternalCatalog)
catalogMgr.getCatalog(dbInfo.getCtlName());
+ if (externalCatalog != null) {
+ externalCatalog.replayCreateDb(dbInfo.getDbName());
+ }
+ }
}
public void dropDb(DropDbStmt stmt) throws DdlException {
@@ -3277,8 +3287,16 @@ public class Env {
catalogIf.dropDb(dbName, ifExists, force);
}
- public void replayDropDb(String dbName, boolean isForceDrop, Long
recycleTime) throws DdlException {
- getInternalCatalog().replayDropDb(dbName, isForceDrop, recycleTime);
+ public void replayDropDb(DropDbInfo info) throws DdlException {
+ if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName()
+ .equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ getInternalCatalog().replayDropDb(info.getDbName(),
info.isForceDrop(), info.getRecycleTime());
+ } else {
+ ExternalCatalog externalCatalog = (ExternalCatalog)
catalogMgr.getCatalog(info.getCtlName());
+ if (externalCatalog != null) {
+ externalCatalog.replayDropDb(info.getDbName());
+ }
+ }
}
public void recoverDatabase(RecoverDbStmt recoverStmt) throws DdlException
{
@@ -4157,8 +4175,16 @@ public class Env {
}
}
- public void replayCreateTable(String dbName, Table table) throws
MetaNotFoundException {
- getInternalCatalog().replayCreateTable(dbName, table);
+ public void replayCreateTable(CreateTableInfo info) throws
MetaNotFoundException {
+ if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName()
+ .equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ getInternalCatalog().replayCreateTable(info.getDbName(),
info.getTable());
+ } else {
+ ExternalCatalog externalCatalog = (ExternalCatalog)
catalogMgr.getCatalog(info.getCtlName());
+ if (externalCatalog != null) {
+ externalCatalog.replayCreateTable(info.getDbName(),
info.getTblName());
+ }
+ }
}
public void replayAlterExternalTableSchema(String dbName, String
tableName, List<Column> newSchema)
@@ -5865,7 +5891,16 @@ public class Env {
}
public void replayTruncateTable(TruncateTableInfo info) throws
MetaNotFoundException {
- getInternalCatalog().replayTruncateTable(info);
+ if (Strings.isNullOrEmpty(info.getCtl()) ||
info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ // In previous versions(before 2.1.8), there is no catalog info in
TruncateTableInfo,
+ // So if the catalog info is empty, we assume it's internal table.
+ getInternalCatalog().replayTruncateTable(info);
+ } else {
+ ExternalCatalog ctl = (ExternalCatalog)
catalogMgr.getCatalog(info.getCtl());
+ if (ctl != null) {
+ ctl.replayTruncateTable(info);
+ }
+ }
}
public void createFunction(CreateFunctionStmt stmt) throws UserException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index 3e6c50ba38b..c28e65c3478 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -189,7 +189,9 @@ public interface CatalogIf<T extends DatabaseIf> {
void createDb(CreateDbStmt stmt) throws DdlException;
- void dropDb(DropDbStmt stmt) throws DdlException;
+ default void dropDb(DropDbStmt stmt) throws DdlException {
+ dropDb(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
+ }
void dropDb(String dbName, boolean ifExists, boolean force) throws
DdlException;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index fc0f674d206..979320815bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -19,7 +19,6 @@ package org.apache.doris.datasource;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
-import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TableRef;
@@ -60,6 +59,11 @@ import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.datasource.test.TestExternalDatabase;
import
org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
+import org.apache.doris.persist.CreateDbInfo;
+import org.apache.doris.persist.CreateTableInfo;
+import org.apache.doris.persist.DropDbInfo;
+import org.apache.doris.persist.DropInfo;
+import org.apache.doris.persist.TruncateTableInfo;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
@@ -934,15 +938,18 @@ public abstract class ExternalCatalog
}
try {
metadataOps.createDb(stmt);
+ CreateDbInfo info = new CreateDbInfo(getName(),
stmt.getFullDbName(), null);
+ Env.getCurrentEnv().getEditLog().logCreateDb(info);
} catch (Exception e) {
- LOG.warn("Failed to create a database.", e);
+ LOG.warn("Failed to create database {} in catalog {}.",
stmt.getFullDbName(), getName(), e);
throw e;
}
}
- @Override
- public void dropDb(DropDbStmt stmt) throws DdlException {
- dropDb(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
+ public void replayCreateDb(String dbName) {
+ if (metadataOps != null) {
+ metadataOps.afterCreateDb(dbName);
+ }
}
@Override
@@ -953,13 +960,21 @@ public abstract class ExternalCatalog
return;
}
try {
- metadataOps.dropDb(dbName, ifExists, force);
+ metadataOps.dropDb(getName(), dbName, ifExists, force);
+ DropDbInfo info = new DropDbInfo(getName(), dbName);
+ Env.getCurrentEnv().getEditLog().logDropDb(info);
} catch (Exception e) {
- LOG.warn("Failed to drop a database.", e);
+ LOG.warn("Failed to drop database {} in catalog {}", dbName,
getName(), e);
throw e;
}
}
+ public void replayDropDb(String dbName) {
+ if (metadataOps != null) {
+ metadataOps.afterDropDb(dbName);
+ }
+ }
+
@Override
public boolean createTable(CreateTableStmt stmt) throws UserException {
makeSureInitialized();
@@ -968,13 +983,25 @@ public abstract class ExternalCatalog
return false;
}
try {
- return metadataOps.createTable(stmt);
+ boolean res = metadataOps.createTable(stmt);
+ if (!res) {
+ // res == false means the table does not exist before, and we
create it.
+ CreateTableInfo info = new CreateTableInfo(getName(),
stmt.getDbName(), stmt.getTableName());
+ Env.getCurrentEnv().getEditLog().logCreateTable(info);
+ }
+ return res;
} catch (Exception e) {
LOG.warn("Failed to create a table.", e);
throw e;
}
}
+ public void replayCreateTable(String dbName, String tblName) {
+ if (metadataOps != null) {
+ metadataOps.afterCreateTable(dbName, tblName);
+ }
+ }
+
@Override
public void dropTable(DropTableStmt stmt) throws DdlException {
makeSureInitialized();
@@ -984,12 +1011,20 @@ public abstract class ExternalCatalog
}
try {
metadataOps.dropTable(stmt);
+ DropInfo info = new DropInfo(getName(), stmt.getDbName(),
stmt.getTableName());
+ Env.getCurrentEnv().getEditLog().logDropTable(info);
} catch (Exception e) {
LOG.warn("Failed to drop a table", e);
throw e;
}
}
+ public void replayDropTable(String dbName, String tblName) {
+ if (metadataOps != null) {
+ metadataOps.afterDropTable(dbName, tblName);
+ }
+ }
+
public void unregisterDatabase(String dbName) {
throw new NotImplementedException("unregisterDatabase not
implemented");
}
@@ -1088,12 +1123,22 @@ public abstract class ExternalCatalog
partitions = tableRef.getPartitionNames().getPartitionNames();
}
metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(),
partitions);
+ TruncateTableInfo info = new TruncateTableInfo(getName(),
tableName.getDb(), tableName.getTbl(),
+ partitions);
+ Env.getCurrentEnv().getEditLog().logTruncateTable(info);
} catch (Exception e) {
- LOG.warn("Failed to drop a table", e);
+ LOG.warn("Failed to truncate table {}.{} in catalog {}",
stmt.getTblRef().getName().getDb(),
+ stmt.getTblRef().getName().getTbl(), getName(), e);
throw e;
}
}
+ public void replayTruncateTable(TruncateTableInfo info) {
+ if (metadataOps != null) {
+ metadataOps.afterTruncateTable(info.getDb(), info.getTable());
+ }
+ }
+
public void setAutoAnalyzePolicy(String dbName, String tableName, String
policy) {
Pair<String, String> key = Pair.of(dbName, tableName);
if (policy == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 1508675b2e2..1795c50d43b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -33,7 +33,6 @@ import org.apache.doris.analysis.CreateTableLikeStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.analysis.DistributionDesc;
-import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropPartitionClause;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.Expr;
@@ -151,6 +150,7 @@ import
org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
import org.apache.doris.persist.AlterDatabasePropertyInfo;
import org.apache.doris.persist.AutoIncrementIdUpdateLog;
import org.apache.doris.persist.ColocatePersistInfo;
+import org.apache.doris.persist.CreateDbInfo;
import org.apache.doris.persist.DatabaseInfo;
import org.apache.doris.persist.DropDbInfo;
import org.apache.doris.persist.DropInfo;
@@ -447,7 +447,8 @@ public class InternalCatalog implements CatalogIf<Database>
{
}
try {
unprotectCreateDb(db);
- Env.getCurrentEnv().getEditLog().logCreateDb(db);
+ CreateDbInfo dbInfo = new
CreateDbInfo(InternalCatalog.INTERNAL_CATALOG_NAME, db.getName(), db);
+ Env.getCurrentEnv().getEditLog().logCreateDb(dbInfo);
} finally {
db.writeUnlock();
}
@@ -486,10 +487,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
}
- public void dropDb(DropDbStmt stmt) throws DdlException {
- dropDb(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
- }
-
+ @Override
public void dropDb(String dbName, boolean ifExists, boolean force) throws
DdlException {
LOG.info("begin drop database[{}], is force : {}", dbName, force);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index 89153a95b72..37005fa0eec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -20,7 +20,6 @@ package org.apache.doris.datasource.hive;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DistributionDesc;
-import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.HashDistributionDesc;
import org.apache.doris.analysis.PartitionDesc;
@@ -107,7 +106,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
}
@Override
- public void createDb(CreateDbStmt stmt) throws DdlException {
+ public void createDbImpl(CreateDbStmt stmt) throws DdlException {
String fullDbName = stmt.getFullDbName();
Map<String, String> properties = stmt.getProperties();
long dbId = Env.getCurrentEnv().getNextId();
@@ -130,7 +129,6 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
catalogDatabase.setProperties(properties);
catalogDatabase.setComment(properties.getOrDefault("comment", ""));
client.createDatabase(catalogDatabase);
- catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
@@ -138,12 +136,12 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
}
@Override
- public void dropDb(DropDbStmt stmt) throws DdlException {
- dropDb(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
+ public void afterCreateDb(String dbName) {
+ catalog.onRefreshCache(true);
}
@Override
- public void dropDb(String dbName, boolean ifExists, boolean force) throws
DdlException {
+ public void dropDbImpl(String dbName, boolean ifExists, boolean force)
throws DdlException {
if (!databaseExist(dbName)) {
if (ifExists) {
LOG.info("drop database[{}] which does not exist", dbName);
@@ -154,14 +152,18 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
}
try {
client.dropDatabase(dbName);
- catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
}
@Override
- public boolean createTable(CreateTableStmt stmt) throws UserException {
+ public void afterDropDb(String dbName) {
+ catalog.onRefreshCache(true);
+ }
+
+ @Override
+ public boolean createTableImpl(CreateTableStmt stmt) throws UserException {
String dbName = stmt.getDbName();
String tblName = stmt.getTableName();
ExternalDatabase<?> db = catalog.getDbNullable(dbName);
@@ -269,7 +271,6 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
comment);
}
client.createTable(hiveTableMeta, stmt.isSetIfNotExists());
- db.setUnInitialized(true);
} catch (Exception e) {
throw new UserException(e.getMessage(), e);
}
@@ -277,7 +278,15 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
}
@Override
- public void dropTable(DropTableStmt stmt) throws DdlException {
+ public void afterCreateTable(String dbName, String tblName) {
+ ExternalDatabase<?> db = catalog.getDbNullable(dbName);
+ if (db != null) {
+ db.setUnInitialized(true);
+ }
+ }
+
+ @Override
+ public void dropTableImpl(DropTableStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
String tblName = stmt.getTableName();
ExternalDatabase<?> db = catalog.getDbNullable(stmt.getDbName());
@@ -303,14 +312,22 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
try {
client.dropTable(dbName, tblName);
- db.setUnInitialized(true);
} catch (Exception e) {
throw new DdlException(e.getMessage(), e);
}
}
@Override
- public void truncateTable(String dbName, String tblName, List<String>
partitions) throws DdlException {
+ public void afterDropTable(String dbName, String tblName) {
+ ExternalDatabase<?> db = catalog.getDbNullable(dbName);
+ if (db != null) {
+ db.setUnInitialized(true);
+ }
+ }
+
+ @Override
+ public void truncateTableImpl(String dbName, String tblName, List<String>
partitions)
+ throws DdlException {
ExternalDatabase<?> db = catalog.getDbNullable(dbName);
if (db == null) {
throw new DdlException("Failed to get database: '" + dbName + "'
in catalog: " + catalog.getName());
@@ -320,9 +337,17 @@ public class HiveMetadataOps implements
ExternalMetadataOps {
} catch (Exception e) {
throw new DdlException(e.getMessage(), e);
}
+ }
+
+ @Override
+ public void afterTruncateTable(String dbName, String tblName) {
+ // Invalidate cache.
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(),
dbName, tblName);
- db.setLastUpdateTime(System.currentTimeMillis());
- db.setUnInitialized(true);
+ ExternalDatabase<?> db = catalog.getDbNullable(dbName);
+ if (db != null) {
+ db.setLastUpdateTime(System.currentTimeMillis());
+ db.setUnInitialized(true);
+ }
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 8c3277316d7..0ee20690ea8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -19,7 +19,6 @@ package org.apache.doris.datasource.iceberg;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
-import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.StructField;
@@ -131,12 +130,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
@Override
- public void createDb(CreateDbStmt stmt) throws DdlException {
+ public void createDbImpl(CreateDbStmt stmt) throws DdlException {
try {
preExecutionAuthenticator.execute(() -> {
performCreateDb(stmt);
return null;
-
});
} catch (Exception e) {
throw new DdlException("Failed to create database: "
@@ -144,6 +142,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
+ @Override
+ public void afterCreateDb(String dbName) {
+ dorisCatalog.onRefreshCache(true);
+ }
+
private void performCreateDb(CreateDbStmt stmt) throws DdlException {
SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
String dbName = stmt.getFullDbName();
@@ -164,16 +167,10 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
nsCatalog.createNamespace(getNamespace(dbName), properties);
- dorisCatalog.onRefreshCache(true);
}
@Override
- public void dropDb(DropDbStmt stmt) throws DdlException {
- dropDb(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
- }
-
- @Override
- public void dropDb(String dbName, boolean ifExists, boolean fore) throws
DdlException {
+ public void dropDbImpl(String dbName, boolean ifExists, boolean fore)
throws DdlException {
try {
preExecutionAuthenticator.execute(() -> {
preformDropDb(dbName, ifExists);
@@ -195,11 +192,15 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
nsCatalog.dropNamespace(getNamespace(dbName));
+ }
+
+ @Override
+ public void afterDropDb(String dbName) {
dorisCatalog.onRefreshCache(true);
}
@Override
- public boolean createTable(CreateTableStmt stmt) throws UserException {
+ public boolean createTableImpl(CreateTableStmt stmt) throws UserException {
try {
preExecutionAuthenticator.execute(() -> performCreateTable(stmt));
} catch (Exception e) {
@@ -236,12 +237,19 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
properties.put(ExternalCatalog.DORIS_VERSION,
ExternalCatalog.DORIS_VERSION_VALUE);
PartitionSpec partitionSpec =
IcebergUtils.solveIcebergPartitionSpec(stmt.getPartitionDesc(), schema);
catalog.createTable(getTableIdentifier(dbName, tableName), schema,
partitionSpec, properties);
- db.setUnInitialized(true);
return false;
}
@Override
- public void dropTable(DropTableStmt stmt) throws DdlException {
+ public void afterCreateTable(String dbName, String tblName) {
+ ExternalDatabase<?> db = dorisCatalog.getDbNullable(dbName);
+ if (db != null) {
+ db.setUnInitialized(true);
+ }
+ }
+
+ @Override
+ public void dropTableImpl(DropTableStmt stmt) throws DdlException {
try {
preExecutionAuthenticator.execute(() -> {
performDropTable(stmt);
@@ -253,6 +261,14 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
+ @Override
+ public void afterDropTable(String dbName, String tblName) {
+ ExternalDatabase<?> db = dorisCatalog.getDbNullable(dbName);
+ if (db != null) {
+ db.setUnInitialized(true);
+ }
+ }
+
private void performDropTable(DropTableStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
String tableName = stmt.getTableName();
@@ -275,11 +291,10 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
catalog.dropTable(getTableIdentifier(dbName, tableName), true);
- db.setUnInitialized(true);
}
@Override
- public void truncateTable(String dbName, String tblName, List<String>
partitions) {
+ public void truncateTableImpl(String dbName, String tblName, List<String>
partitions) {
throw new UnsupportedOperationException("Truncate Iceberg table is not
supported.");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
index b8d434b95e1..e8d6842ac44 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
@@ -36,14 +36,30 @@ public interface ExternalMetadataOps {
* @param stmt
* @throws DdlException
*/
- void createDb(CreateDbStmt stmt) throws DdlException;
+ default void createDb(CreateDbStmt stmt) throws DdlException {
+ createDbImpl(stmt);
+ afterCreateDb(stmt.getFullDbName());
+ }
+
+ void createDbImpl(CreateDbStmt stmt) throws DdlException;
+
+ default void afterCreateDb(String dbName) {
+ }
/**
* drop db in external metastore
* @param stmt
* @throws DdlException
*/
- void dropDb(DropDbStmt stmt) throws DdlException;
+ default void dropDb(DropDbStmt stmt) throws DdlException {
+ dropDbImpl(stmt.getDbName(), stmt.isSetIfExists(), stmt.isForceDrop());
+ afterDropDb(stmt.getCtlName());
+ }
+
+ default void dropDb(String ctlName, String dbName, boolean ifExists,
boolean force) throws DdlException {
+ dropDbImpl(dbName, ifExists, force);
+ afterDropDb(ctlName);
+ }
/**
* drop db in external metastore for nereids
@@ -52,7 +68,9 @@ public interface ExternalMetadataOps {
* @param force
* @throws DdlException
*/
- void dropDb(String dbName, boolean ifExists, boolean force) throws
DdlException;
+ void dropDbImpl(String dbName, boolean ifExists, boolean force) throws
DdlException;
+
+ void afterDropDb(String dbName);
/**
*
@@ -60,14 +78,33 @@ public interface ExternalMetadataOps {
* @return if set isExists is true, return true if table exists, otherwise
return false
* @throws UserException
*/
- boolean createTable(CreateTableStmt stmt) throws UserException;
+ default boolean createTable(CreateTableStmt stmt) throws UserException {
+ boolean res = createTableImpl(stmt);
+ if (!res) {
+ afterCreateTable(stmt.getDbName(), stmt.getTableName());
+ }
+ return res;
+ }
+
+ boolean createTableImpl(CreateTableStmt stmt) throws UserException;
+
+ default void afterCreateTable(String dbName, String tblName) {
+ }
/**
*
* @param stmt
* @throws DdlException
*/
- void dropTable(DropTableStmt stmt) throws DdlException;
+ default void dropTable(DropTableStmt stmt) throws DdlException {
+ dropTableImpl(stmt);
+ afterDropTable(stmt.getDbName(), stmt.getTableName());
+ }
+
+ void dropTableImpl(DropTableStmt stmt) throws DdlException;
+
+ default void afterDropTable(String dbName, String tblName) {
+ }
/**
*
@@ -75,7 +112,15 @@ public interface ExternalMetadataOps {
* @param tblName
* @param partitions
*/
- void truncateTable(String dbName, String tblName, List<String> partitions)
throws DdlException;
+ default void truncateTable(String dbName, String tblName, List<String>
partitions) throws DdlException {
+ truncateTableImpl(dbName, tblName, partitions);
+ afterTruncateTable(dbName, tblName);
+ }
+
+ void truncateTableImpl(String dbName, String tblName, List<String>
partitions) throws DdlException;
+
+ default void afterTruncateTable(String dbName, String tblName) {
+ }
/**
*
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index 7527755baab..29096b68ce0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -81,6 +81,7 @@ import org.apache.doris.persist.CleanLabelOperationLog;
import org.apache.doris.persist.CleanQueryStatsInfo;
import org.apache.doris.persist.ColocatePersistInfo;
import org.apache.doris.persist.ConsistencyCheckInfo;
+import org.apache.doris.persist.CreateDbInfo;
import org.apache.doris.persist.CreateTableInfo;
import org.apache.doris.persist.DatabaseInfo;
import org.apache.doris.persist.DropDbInfo;
@@ -217,6 +218,11 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
+ case OperationType.OP_NEW_CREATE_DB: {
+ data = CreateDbInfo.read(in);
+ isRead = true;
+ break;
+ }
case OperationType.OP_DROP_DB: {
data = DropDbInfo.read(in);
isRead = true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java
new file mode 100644
index 00000000000..522ae021429
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class CreateDbInfo implements Writable {
+
+ @SerializedName("ctl")
+ private String ctlName;
+ @SerializedName("db")
+ private String dbName;
+ @SerializedName("idb")
+ private Database internalDb;
+
+ public CreateDbInfo() {
+ this.ctlName = "";
+ this.dbName = "";
+ }
+
+ public CreateDbInfo(String ctlName, String dbName, Database internalDb) {
+ this.ctlName = ctlName;
+ this.dbName = dbName;
+ this.internalDb = internalDb;
+ }
+
+ public String getCtlName() {
+ return ctlName;
+ }
+
+ public String getDbName() {
+ return dbName;
+ }
+
+ public Database getInternalDb() {
+ return internalDb;
+ }
+
+ public static CreateDbInfo read(DataInput in) throws IOException {
+ String json = Text.readString(in);
+ return GsonUtils.GSON.fromJson(json, CreateDbInfo.class);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ String json = GsonUtils.GSON.toJson(this);
+ Text.writeString(out, json);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java
index 1f2bcc15eb4..eab2f2b872e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java
@@ -23,9 +23,11 @@ import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
+import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,8 +40,12 @@ import java.util.Objects;
public class CreateTableInfo implements Writable, GsonPostProcessable {
public static final Logger LOG =
LoggerFactory.getLogger(CreateTableInfo.class);
+ @SerializedName(value = "ctl")
+ private String ctlName;
@SerializedName(value = "dbName")
private String dbName;
+ @SerializedName(value = "tbl")
+ private String tblName;
@SerializedName(value = "table")
private Table table;
@@ -47,15 +53,33 @@ public class CreateTableInfo implements Writable,
GsonPostProcessable {
// for persist
}
+ // for internal table
public CreateTableInfo(String dbName, Table table) {
+ this.ctlName = InternalCatalog.INTERNAL_CATALOG_NAME;
this.dbName = dbName;
+ this.tblName = table.getName();
this.table = table;
}
+ // for external table
+ public CreateTableInfo(String ctlName, String dbName, String tblName) {
+ this.ctlName = ctlName;
+ this.dbName = dbName;
+ this.tblName = tblName;
+ }
+
+ public String getCtlName() {
+ return ctlName;
+ }
+
public String getDbName() {
return dbName;
}
+ public String getTblName() {
+ return tblName;
+ }
+
public Table getTable() {
return table;
}
@@ -106,7 +130,11 @@ public class CreateTableInfo implements Writable,
GsonPostProcessable {
@Override
public String toString() {
- return toJson();
+ // In previous versions, ctlName and tblName is not set, so it may be
null.
+ return String.format("%s.%s.%s",
+ Strings.isNullOrEmpty(ctlName) ?
InternalCatalog.INTERNAL_CATALOG_NAME : ctlName,
+ dbName,
+ Strings.isNullOrEmpty(tblName) ? table.getName() : tblName);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
index 190db6a671a..f9d7339ca41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
@@ -30,6 +30,8 @@ import java.io.DataOutput;
import java.io.IOException;
public class DropDbInfo implements Writable, GsonPostProcessable {
+ @SerializedName(value = "ctl")
+ private String ctlName;
@SerializedName(value = "dbName")
private String dbName;
@SerializedName(value = "forceDrop")
@@ -41,12 +43,23 @@ public class DropDbInfo implements Writable,
GsonPostProcessable {
this("", false, 0);
}
+ // for external table
+ public DropDbInfo(String ctlName, String dbName) {
+ this.ctlName = ctlName;
+ this.dbName = dbName;
+ }
+
+ // for internal table
public DropDbInfo(String dbName, boolean forceDrop, long recycleTime) {
this.dbName = dbName;
this.forceDrop = forceDrop;
this.recycleTime = recycleTime;
}
+ public String getCtlName() {
+ return ctlName;
+ }
+
public String getDbName() {
return dbName;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
index db0688bd6ad..ff80f6ecb17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
@@ -21,8 +21,10 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.persist.gson.GsonUtils;
+import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
@@ -30,6 +32,10 @@ import java.io.DataOutput;
import java.io.IOException;
public class DropInfo implements Writable {
+ @SerializedName(value = "ctl")
+ private String ctl;
+ @SerializedName(value = "db")
+ private String db;
@SerializedName(value = "dbId")
private long dbId;
@SerializedName(value = "tableId")
@@ -50,13 +56,23 @@ public class DropInfo implements Writable {
public DropInfo() {
}
+ // for external table
+ public DropInfo(String ctl, String db, String tbl) {
+ this.ctl = ctl;
+ this.db = db;
+ this.tableName = tbl;
+ }
+
+ // for internal table
public DropInfo(long dbId, long tableId, String tableName, boolean isView,
boolean forceDrop,
long recycleTime) {
this(dbId, tableId, tableName, -1L, "", isView, forceDrop,
recycleTime);
}
+ // for internal table
public DropInfo(long dbId, long tableId, String tableName, long indexId,
String indexName, boolean isView,
boolean forceDrop, long recycleTime) {
+ this.ctl = InternalCatalog.INTERNAL_CATALOG_NAME;
this.dbId = dbId;
this.tableId = tableId;
this.tableName = tableName;
@@ -67,6 +83,14 @@ public class DropInfo implements Writable {
this.recycleTime = recycleTime;
}
+ public String getCtl() {
+ return ctl;
+ }
+
+ public String getDb() {
+ return db;
+ }
+
public long getDbId() {
return this.dbId;
}
@@ -149,4 +173,13 @@ public class DropInfo implements Writable {
public static DropInfo fromJson(String json) {
return GsonUtils.GSON.fromJson(json, DropInfo.class);
}
+
+ @Override
+ public String toString() {
+ // In previous versions, ctl and db are not set, so they may be null.
+ return String.format("%s.%s.%s",
+ Strings.isNullOrEmpty(ctl) ?
InternalCatalog.INTERNAL_CATALOG_NAME : ctl,
+ Strings.isNullOrEmpty(db) ? dbId : db,
+ tableName);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 7b719c0fb0f..29c036a42d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -51,9 +51,11 @@ import org.apache.doris.cooldown.CooldownConfHandler;
import org.apache.doris.cooldown.CooldownConfList;
import org.apache.doris.cooldown.CooldownDelete;
import org.apache.doris.datasource.CatalogLog;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.InitDatabaseLog;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.MetaIdMappingsLog;
import org.apache.doris.ha.MasterInfo;
import org.apache.doris.insertoverwrite.InsertOverwriteLog;
@@ -100,6 +102,7 @@ import org.apache.doris.system.Frontend;
import org.apache.doris.transaction.TransactionState;
import org.apache.doris.transaction.TransactionStatus;
+import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -191,12 +194,18 @@ public class EditLog {
}
case OperationType.OP_CREATE_DB: {
Database db = (Database) journal.getData();
- env.replayCreateDb(db);
+ CreateDbInfo info = new
CreateDbInfo(db.getCatalog().getName(), db.getName(), db);
+ env.replayCreateDb(info);
+ break;
+ }
+ case OperationType.OP_NEW_CREATE_DB: {
+ CreateDbInfo info = (CreateDbInfo) journal.getData();
+ env.replayCreateDb(info);
break;
}
case OperationType.OP_DROP_DB: {
DropDbInfo dropDbInfo = (DropDbInfo) journal.getData();
- env.replayDropDb(dropDbInfo.getDbName(),
dropDbInfo.isForceDrop(), dropDbInfo.getRecycleTime());
+ env.replayDropDb(dropDbInfo);
break;
}
case OperationType.OP_ALTER_DB: {
@@ -225,28 +234,38 @@ public class EditLog {
}
case OperationType.OP_CREATE_TABLE: {
CreateTableInfo info = (CreateTableInfo) journal.getData();
- LOG.info("Begin to unprotect create table. db = " +
info.getDbName() + " table = " + info.getTable()
- .getId());
- CreateTableRecord record = new CreateTableRecord(logId,
info);
- env.replayCreateTable(info.getDbName(), info.getTable());
- env.getBinlogManager().addCreateTableRecord(record);
+ LOG.info("Begin to unprotect create table. {}", info);
+ env.replayCreateTable(info);
+ if (Strings.isNullOrEmpty(info.getCtlName()) ||
info.getCtlName().equals(
+ InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ CreateTableRecord record = new
CreateTableRecord(logId, info);
+ env.getBinlogManager().addCreateTableRecord(record);
+ }
break;
}
case OperationType.OP_ALTER_EXTERNAL_TABLE_SCHEMA: {
RefreshExternalTableInfo info = (RefreshExternalTableInfo)
journal.getData();
- LOG.info("Begin to unprotect alter external table schema.
db = " + info.getDbName() + " table = "
- + info.getTableName());
+ LOG.info("Begin to unprotect alter external table schema.
db = {} table = {}", info.getDbName(),
+ info.getTableName());
env.replayAlterExternalTableSchema(info.getDbName(),
info.getTableName(), info.getNewSchema());
break;
}
case OperationType.OP_DROP_TABLE: {
DropInfo info = (DropInfo) journal.getData();
- Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId());
- LOG.info("Begin to unprotect drop table. db = " +
db.getFullName() + " table = "
- + info.getTableId());
- DropTableRecord record = new DropTableRecord(logId, info);
- env.replayDropTable(db, info.getTableId(),
info.isForceDrop(), info.getRecycleTime());
- env.getBinlogManager().addDropTableRecord(record);
+ LOG.info("Begin to unprotect drop table: {}", info);
+ if (Strings.isNullOrEmpty(info.getCtl()) ||
info.getCtl().equals(
+ InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId());
+ env.replayDropTable(db, info.getTableId(),
info.isForceDrop(), info.getRecycleTime());
+ DropTableRecord record = new DropTableRecord(logId,
info);
+ env.getBinlogManager().addDropTableRecord(record);
+ } else {
+ ExternalCatalog ctl = (ExternalCatalog)
Env.getCurrentEnv().getCatalogMgr()
+ .getCatalog(info.getCtl());
+ if (ctl != null) {
+ ctl.replayDropTable(info.getDb(),
info.getTableName());
+ }
+ }
break;
}
case OperationType.OP_ADD_PARTITION: {
@@ -1406,8 +1425,8 @@ public class EditLog {
logEdit(OperationType.OP_SAVE_TRANSACTION_ID, new
Text(Long.toString(transactionId)));
}
- public void logCreateDb(Database db) {
- logEdit(OperationType.OP_CREATE_DB, db);
+ public void logCreateDb(CreateDbInfo info) {
+ logEdit(OperationType.OP_NEW_CREATE_DB, info);
}
public void logDropDb(DropDbInfo dropDbInfo) {
@@ -1428,8 +1447,11 @@ public class EditLog {
public void logCreateTable(CreateTableInfo info) {
long logId = logEdit(OperationType.OP_CREATE_TABLE, info);
- CreateTableRecord record = new CreateTableRecord(logId, info);
- Env.getCurrentEnv().getBinlogManager().addCreateTableRecord(record);
+ if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName()
+ .equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ CreateTableRecord record = new CreateTableRecord(logId, info);
+
Env.getCurrentEnv().getBinlogManager().addCreateTableRecord(record);
+ }
}
public void logRefreshExternalTableSchema(RefreshExternalTableInfo info) {
@@ -1473,8 +1495,10 @@ public class EditLog {
public void logDropTable(DropInfo info) {
long logId = logEdit(OperationType.OP_DROP_TABLE, info);
- DropTableRecord record = new DropTableRecord(logId, info);
- Env.getCurrentEnv().getBinlogManager().addDropTableRecord(record);
+ if (Strings.isNullOrEmpty(info.getCtl()) ||
info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ DropTableRecord record = new DropTableRecord(logId, info);
+ Env.getCurrentEnv().getBinlogManager().addDropTableRecord(record);
+ }
}
public void logEraseTable(long tableId) {
@@ -1712,7 +1736,9 @@ public class EditLog {
public void logTruncateTable(TruncateTableInfo info) {
long logId = logEdit(OperationType.OP_TRUNCATE_TABLE, info);
LOG.info("log truncate table, logId:{}, infos: {}", logId, info);
- Env.getCurrentEnv().getBinlogManager().addTruncateTable(info, logId);
+ if (Strings.isNullOrEmpty(info.getCtl()) ||
info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ Env.getCurrentEnv().getBinlogManager().addTruncateTable(info,
logId);
+ }
}
public void logColocateModifyRepliaAlloc(ColocatePersistInfo info) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index 6c97606cb24..8636a3dbc5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -27,12 +27,14 @@ public class OperationType {
// OP_LOCAL_EOF is only for local edit log, to indicate the end of a edit
log run.
public static final short OP_LOCAL_EOF = -1;
public static final short OP_SAVE_NEXTID = 0;
- public static final short OP_CREATE_DB = 1;
+ @Deprecated
+ public static final short OP_CREATE_DB = 1; // deprecated, use
OP_NEW_CREATE_DB instead
public static final short OP_DROP_DB = 2;
public static final short OP_ALTER_DB = 3;
public static final short OP_ERASE_DB = 4;
public static final short OP_RECOVER_DB = 5;
public static final short OP_RENAME_DB = 6;
+ public static final short OP_NEW_CREATE_DB = 7;
// 10~19 110~119 210~219 ...
public static final short OP_CREATE_TABLE = 10;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
index ffb59d2afdf..10d1e07074a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
@@ -33,6 +33,8 @@ import java.util.List;
import java.util.Map;
public class TruncateTableInfo implements Writable {
+ @SerializedName(value = "ctl")
+ private String ctl;
@SerializedName(value = "dbId")
private long dbId;
@SerializedName(value = "db")
@@ -43,6 +45,9 @@ public class TruncateTableInfo implements Writable {
private String table;
@SerializedName(value = "partitions")
private List<Partition> partitions = Lists.newArrayList();
+ // Only for external table
+ @SerializedName(value = "extParts")
+ private List<String> extPartNames = Lists.newArrayList();
@SerializedName(value = "isEntireTable")
private boolean isEntireTable = false;
@SerializedName(value = "rawSql")
@@ -56,6 +61,7 @@ public class TruncateTableInfo implements Writable {
}
+ // for internal table
public TruncateTableInfo(long dbId, String db, long tblId, String table,
List<Partition> partitions,
boolean isEntireTable, String rawSql, List<Partition>
oldPartitions, boolean force) {
this.dbId = dbId;
@@ -71,6 +77,18 @@ public class TruncateTableInfo implements Writable {
this.force = force;
}
+ // for external table
+ public TruncateTableInfo(String ctl, String db, String table, List<String>
partNames) {
+ this.ctl = ctl;
+ this.db = db;
+ this.table = table;
+ this.extPartNames = partNames;
+ }
+
+ public String getCtl() {
+ return ctl;
+ }
+
public long getDbId() {
return dbId;
}
@@ -91,6 +109,10 @@ public class TruncateTableInfo implements Writable {
return partitions;
}
+ public List<String> getExtPartNames() {
+ return extPartNames;
+ }
+
public Map<Long, String> getOldPartitions() {
return oldPartitions == null ? new HashMap<>() : oldPartitions;
}
@@ -103,7 +125,6 @@ public class TruncateTableInfo implements Writable {
return force;
}
-
public String getRawSql() {
return rawSql;
}
@@ -126,13 +147,15 @@ public class TruncateTableInfo implements Writable {
@Override
public String toString() {
return "TruncateTableInfo{"
+ + "ctl=" + ctl
+ "dbId=" + dbId
+ ", db='" + db + '\''
+ ", tblId=" + tblId
+ ", table='" + table + '\''
+ ", isEntireTable=" + isEntireTable
+ ", rawSql='" + rawSql + '\''
- + ", partitions_size=" + partitions.size()
+ + ", partitions_size=" + (partitions == null ? "0" :
partitions.size())
+ + ", extPartNames_size=" + (extPartNames == null ? "0" :
extPartNames.size())
+ '}';
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java
new file mode 100644
index 00000000000..881c61065be
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/CreateDbInfoTest.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.meta.MetaContext;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.nio.file.Files;
+
+public class CreateDbInfoTest {
+ @Test
+ public void testSerialization() throws Exception {
+ MetaContext metaContext = new MetaContext();
+ metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
+ metaContext.setThreadLocalInfo();
+
+ // 1. Write objects to file
+ File file = new File("./createDbInfo");
+ file.createNewFile();
+ DataOutputStream dos = new
DataOutputStream(Files.newOutputStream(file.toPath()));
+
+ Database db = new Database(10000, "db1");
+ CreateDbInfo info1 = new
CreateDbInfo(InternalCatalog.INTERNAL_CATALOG_NAME, db.getName(), db);
+ info1.write(dos);
+
+ CreateDbInfo info2 = new CreateDbInfo("external_catalog",
"external_db", null);
+ info2.write(dos);
+
+ dos.flush();
+ dos.close();
+
+ // 2. Read objects from file
+ DataInputStream dis = new
DataInputStream(Files.newInputStream(file.toPath()));
+
+ CreateDbInfo rInfo1 = CreateDbInfo.read(dis);
+ Assert.assertEquals(info1.getCtlName(), rInfo1.getCtlName());
+ Assert.assertEquals(info1.getDbName(), rInfo1.getDbName());
+ Assert.assertEquals(info1.getInternalDb().getId(),
rInfo1.getInternalDb().getId());
+
+ CreateDbInfo rInfo2 = CreateDbInfo.read(dis);
+ Assert.assertEquals(info2.getCtlName(), rInfo2.getCtlName());
+ Assert.assertEquals(info2.getDbName(), rInfo2.getDbName());
+ Assert.assertNull(rInfo2.getInternalDb());
+
+ // 3. delete files
+ dis.close();
+ file.delete();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
