This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new a175e041d29 branch-4.0: [fix](catalog) update the table's last update
time after related operations. (#59387) (#59695)
a175e041d29 is described below
commit a175e041d29f7575ec6dbe993cd4eefdb6e3ba1d
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Jan 12 09:29:54 2026 +0800
branch-4.0: [fix](catalog) update the table's last update time after
related operations. (#59387) (#59695)
bp (#59387)
---
.../main/java/org/apache/doris/alter/Alter.java | 7 +--
.../org/apache/doris/catalog/RefreshManager.java | 19 ++++---
.../org/apache/doris/datasource/CatalogMgr.java | 4 +-
.../apache/doris/datasource/ExternalCatalog.java | 41 +++++++++------
.../apache/doris/datasource/ExternalObjectLog.java | 7 ++-
.../org/apache/doris/datasource/ExternalTable.java | 21 +++-----
.../doris/datasource/hive/HMSExternalDatabase.java | 2 +-
.../doris/datasource/hive/HMSExternalTable.java | 18 +------
.../doris/datasource/hive/HiveMetadataOps.java | 4 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 20 +++----
.../datasource/iceberg/IcebergMetadataOps.java | 43 ++++++++-------
.../datasource/operations/ExternalMetadataOps.java | 18 +++----
.../doris/datasource/test/TestExternalTable.java | 1 +
.../trees/plans/commands/ExecuteActionCommand.java | 6 +--
.../plans/commands/insert/HiveInsertExecutor.java | 7 ++-
.../apache/doris/persist/TruncateTableInfo.java | 9 +++-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 2 +-
.../org/apache/doris/catalog/RefreshTableTest.java | 16 +++---
.../apache/doris/external/hms/HmsCatalogTest.java | 15 ++----
.../org/apache/doris/qe/HmsQueryCacheTest.java | 12 ++---
.../iceberg/iceberg_branch_tag_operate.groovy | 13 ++++-
.../iceberg/iceberg_schema_change_ddl.groovy | 7 +++
.../test_external_table_update_time.groovy | 61 ++++++++++++++++++++++
23 files changed, 218 insertions(+), 135 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 9150c60879d..1c3f94cce68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -392,6 +392,7 @@ public class Alter {
private void processAlterTableForExternalTable(
ExternalTable table,
List<AlterClause> alterClauses) throws UserException {
+ long updateTime = System.currentTimeMillis();
for (AlterClause alterClause : alterClauses) {
if (alterClause instanceof ModifyTablePropertiesClause) {
setExternalTableAutoAnalyzePolicy(table, alterClauses);
@@ -438,7 +439,7 @@ public class Alter {
AddPartitionFieldClause addPartitionField =
(AddPartitionFieldClause) alterClause;
if (table instanceof IcebergExternalTable) {
((IcebergExternalCatalog)
table.getCatalog()).addPartitionField(
- (IcebergExternalTable) table, addPartitionField);
+ (IcebergExternalTable) table, addPartitionField,
updateTime);
} else {
throw new UserException("ADD PARTITION KEY is only
supported for Iceberg tables");
}
@@ -446,7 +447,7 @@ public class Alter {
DropPartitionFieldClause dropPartitionField =
(DropPartitionFieldClause) alterClause;
if (table instanceof IcebergExternalTable) {
((IcebergExternalCatalog)
table.getCatalog()).dropPartitionField(
- (IcebergExternalTable) table, dropPartitionField);
+ (IcebergExternalTable) table, dropPartitionField,
updateTime);
} else {
throw new UserException("DROP PARTITION KEY is only
supported for Iceberg tables");
}
@@ -454,7 +455,7 @@ public class Alter {
ReplacePartitionFieldClause replacePartitionField =
(ReplacePartitionFieldClause) alterClause;
if (table instanceof IcebergExternalTable) {
((IcebergExternalCatalog)
table.getCatalog()).replacePartitionField(
- (IcebergExternalTable) table,
replacePartitionField);
+ (IcebergExternalTable) table,
replacePartitionField, updateTime);
} else {
throw new UserException("REPLACE PARTITION KEY is only
supported for Iceberg tables");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 2802055a887..54237f2d38e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -146,10 +146,10 @@ public class RefreshManager {
}
return;
}
- refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, 0);
-
+ long updateTime = System.currentTimeMillis();
+ refreshTableInternal((ExternalDatabase) db, (ExternalTable) table,
updateTime);
ExternalObjectLog log =
ExternalObjectLog.createForRefreshTable(catalog.getId(), db.getFullName(),
- table.getName());
+ table.getName(), updateTime);
Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log);
}
@@ -194,6 +194,9 @@ public class RefreshManager {
HiveMetaStoreCache cache =
Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) catalog);
cache.refreshAffectedPartitionsCache((HMSExternalTable)
table.get(), modifiedPartNames, newPartNames);
+ if (table.get() instanceof HMSExternalTable &&
log.getLastUpdateTime() > 0) {
+ ((HMSExternalTable)
table.get()).setUpdateTime(log.getLastUpdateTime());
+ }
LOG.info("replay refresh partitions for table {}, "
+ "modified partitions count: {}, "
+ "new partitions count: {}",
@@ -229,12 +232,12 @@ public class RefreshManager {
public void refreshTableInternal(ExternalDatabase db, ExternalTable table,
long updateTime) {
table.unsetObjectCreated();
- if (table instanceof HMSExternalTable && updateTime > 0) {
- ((HMSExternalTable) table).setEventUpdateTime(updateTime);
+ if (updateTime > 0) {
+ table.setUpdateTime(updateTime);
}
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table);
- LOG.info("refresh table {}, id {} from db {} in catalog {}",
- table.getName(), table.getId(), db.getFullName(),
db.getCatalog().getName());
+ LOG.info("refresh table {}, id {} from db {} in catalog {}, update
time: {}",
+ table.getName(), table.getId(), db.getFullName(),
db.getCatalog().getName(), updateTime);
}
// Refresh partition
@@ -268,7 +271,7 @@ public class RefreshManager {
}
Env.getCurrentEnv().getExtMetaCacheMgr().invalidatePartitionsCache((ExternalTable)
table, partitionNames);
- ((HMSExternalTable) table).setEventUpdateTime(updateTime);
+ ((HMSExternalTable) table).setUpdateTime(updateTime);
}
public void addToRefreshMap(long catalogId, Integer[] sec) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 692d32c5931..16eeebd7dff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -726,7 +726,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
HMSExternalTable hmsTable = (HMSExternalTable) table;
Env.getCurrentEnv().getExtMetaCacheMgr().addPartitionsCache(catalog.getId(),
hmsTable, partitionNames);
- hmsTable.setEventUpdateTime(updateTime);
+ hmsTable.setUpdateTime(updateTime);
}
public void dropExternalPartitions(String catalogName, String dbName,
String tableName,
@@ -757,7 +757,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
HMSExternalTable hmsTable = (HMSExternalTable) table;
Env.getCurrentEnv().getExtMetaCacheMgr().dropPartitionsCache(catalog.getId(),
hmsTable, partitionNames);
- hmsTable.setEventUpdateTime(updateTime);
+ hmsTable.setUpdateTime(updateTime);
}
public void registerCatalogRefreshListener(Env env) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 629af2c2e26..c2f5a3e9c70 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -1137,8 +1137,9 @@ public abstract class ExternalCatalog
partitions = partitionNamesInfo.getPartitionNames();
}
ExternalTable dorisTable =
getDbOrDdlException(dbName).getTableOrDdlException(tableName);
- metadataOps.truncateTable(dorisTable, partitions);
- TruncateTableInfo info = new TruncateTableInfo(getName(), dbName,
tableName, partitions);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.truncateTable(dorisTable, partitions, updateTime);
+ TruncateTableInfo info = new TruncateTableInfo(getName(), dbName,
tableName, partitions, updateTime);
Env.getCurrentEnv().getEditLog().logTruncateTable(info);
} catch (Exception e) {
LOG.warn("Failed to truncate table {}.{} in catalog {}", dbName,
tableName, getName(), e);
@@ -1148,7 +1149,7 @@ public abstract class ExternalCatalog
public void replayTruncateTable(TruncateTableInfo info) {
if (metadataOps != null) {
- metadataOps.afterTruncateTable(info.getDb(), info.getTable());
+ metadataOps.afterTruncateTable(info.getDb(), info.getTable(),
info.getUpdateTime());
}
}
@@ -1320,11 +1321,11 @@ public abstract class ExternalCatalog
}
// log the refresh external table operation
- private void logRefreshExternalTable(ExternalTable dorisTable) {
+ private void logRefreshExternalTable(ExternalTable dorisTable, long
updateTime) {
Env.getCurrentEnv().getEditLog()
.logRefreshExternalTable(
ExternalObjectLog.createForRefreshTable(dorisTable.getCatalog().getId(),
- dorisTable.getDbName(), dorisTable.getName()));
+ dorisTable.getDbName(), dorisTable.getName(),
updateTime));
}
@Override
@@ -1336,8 +1337,9 @@ public abstract class ExternalCatalog
throw new DdlException("Add column operation is not supported for
catalog: " + getName());
}
try {
- metadataOps.addColumn(externalTable, column, position);
- logRefreshExternalTable(externalTable);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.addColumn(externalTable, column, position, updateTime);
+ logRefreshExternalTable(externalTable, updateTime);
} catch (Exception e) {
LOG.warn("Failed to add column {} to table {}.{} in catalog {}",
column.getName(), externalTable.getDbName(),
externalTable.getName(), getName(), e);
@@ -1354,8 +1356,9 @@ public abstract class ExternalCatalog
throw new DdlException("Add columns operation is not supported for
catalog: " + getName());
}
try {
- metadataOps.addColumns(externalTable, columns);
- logRefreshExternalTable(externalTable);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.addColumns(externalTable, columns, updateTime);
+ logRefreshExternalTable(externalTable, updateTime);
} catch (Exception e) {
LOG.warn("Failed to add columns to table {}.{} in catalog {}",
externalTable.getDbName(), externalTable.getName(),
getName(), e);
@@ -1372,8 +1375,9 @@ public abstract class ExternalCatalog
throw new DdlException("Drop column operation is not supported for
catalog: " + getName());
}
try {
- metadataOps.dropColumn(externalTable, columnName);
- logRefreshExternalTable(externalTable);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.dropColumn(externalTable, columnName, updateTime);
+ logRefreshExternalTable(externalTable, updateTime);
} catch (Exception e) {
LOG.warn("Failed to drop column {} from table {}.{} in catalog {}",
columnName, externalTable.getDbName(),
externalTable.getName(), getName(), e);
@@ -1390,8 +1394,9 @@ public abstract class ExternalCatalog
throw new DdlException("Rename column operation is not supported
for catalog: " + getName());
}
try {
- metadataOps.renameColumn(externalTable, oldName, newName);
- logRefreshExternalTable(externalTable);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.renameColumn(externalTable, oldName, newName,
updateTime);
+ logRefreshExternalTable(externalTable, updateTime);
} catch (Exception e) {
LOG.warn("Failed to rename column {} to {} in table {}.{} in
catalog {}",
oldName, newName, externalTable.getDbName(),
externalTable.getName(), getName(), e);
@@ -1408,8 +1413,9 @@ public abstract class ExternalCatalog
throw new DdlException("Modify column operation is not supported
for catalog: " + getName());
}
try {
- metadataOps.modifyColumn(externalTable, column, columnPosition);
- logRefreshExternalTable(externalTable);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.modifyColumn(externalTable, column, columnPosition,
updateTime);
+ logRefreshExternalTable(externalTable, updateTime);
} catch (Exception e) {
LOG.warn("Failed to modify column {} in table {}.{} in catalog {}",
column.getName(), externalTable.getDbName(),
externalTable.getName(), getName(), e);
@@ -1426,8 +1432,9 @@ public abstract class ExternalCatalog
throw new DdlException("Reorder columns operation is not supported
for catalog: " + getName());
}
try {
- metadataOps.reorderColumns(externalTable, newOrder);
- logRefreshExternalTable(externalTable);
+ long updateTime = System.currentTimeMillis();
+ metadataOps.reorderColumns(externalTable, newOrder, updateTime);
+ logRefreshExternalTable(externalTable, updateTime);
} catch (Exception e) {
LOG.warn("Failed to reorder columns in table {}.{} in catalog {}",
externalTable.getDbName(), externalTable.getName(),
getName(), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
index 43a0675d783..84f0605b8d8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
@@ -75,22 +75,25 @@ public class ExternalObjectLog implements Writable {
return externalObjectLog;
}
- public static ExternalObjectLog createForRefreshTable(long catalogId,
String dbName, String tblName) {
+ public static ExternalObjectLog createForRefreshTable(long catalogId,
String dbName, String tblName,
+ long updateTime) {
ExternalObjectLog externalObjectLog = new ExternalObjectLog();
externalObjectLog.setCatalogId(catalogId);
externalObjectLog.setDbName(dbName);
externalObjectLog.setTableName(tblName);
+ externalObjectLog.setLastUpdateTime(updateTime);
return externalObjectLog;
}
public static ExternalObjectLog createForRefreshPartitions(long catalogId,
String dbName, String tblName,
- List<String> modifiedPartNames, List<String> newPartNames) {
+ List<String> modifiedPartNames, List<String> newPartNames, long
updateTime) {
ExternalObjectLog externalObjectLog = new ExternalObjectLog();
externalObjectLog.setCatalogId(catalogId);
externalObjectLog.setDbName(dbName);
externalObjectLog.setTableName(tblName);
externalObjectLog.setPartitionNames(modifiedPartNames);
externalObjectLog.setNewPartitionNames(newPartNames);
+ externalObjectLog.setLastUpdateTime(updateTime);
return externalObjectLog;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index cf74ef2fb54..4a81bb3c7f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -85,8 +85,8 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
@SerializedName(value = "ta")
private final TableAttributes tableAttributes = new TableAttributes();
- // this field will be refreshed after reloading schema
- protected volatile long schemaUpdateTime;
+ // record the table update time, like insert/alter/delete
+ protected volatile long updateTime = 0;
protected long dbId;
protected boolean objectCreated;
@@ -276,16 +276,15 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
return 0;
}
- // return schema update time as default
- // override this method if there is some other kinds of update time
- // use getSchemaUpdateTime if just need the schema update time
@Override
+ // Returns the table update time, tracking when the table was last modified
+ // (for example, by insert, alter, or refresh operations).
public long getUpdateTime() {
- return this.schemaUpdateTime;
+ return updateTime;
}
- public void setUpdateTime(long schemaUpdateTime) {
- this.schemaUpdateTime = schemaUpdateTime;
+ public void setUpdateTime(long updateTime) {
+ this.updateTime = updateTime;
}
@Override
@@ -345,7 +344,7 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
* @return
*/
public Optional<SchemaCacheValue> initSchemaAndUpdateTime(SchemaCacheKey
key) {
- schemaUpdateTime = System.currentTimeMillis();
+ setUpdateTime(System.currentTimeMillis());
return initSchema(key);
}
@@ -484,10 +483,6 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
return Objects.hashCode(name, db);
}
- public long getSchemaUpdateTime() {
- return schemaUpdateTime;
- }
-
public long getDbId() {
return dbId;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
index 467887fd476..ab0884d0ce1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
@@ -51,7 +51,7 @@ public class HMSExternalDatabase extends
ExternalDatabase<HMSExternalTable> {
super.registerTable(tableIf);
HMSExternalTable table = getTableNullable(tableIf.getName());
if (table != null) {
- table.setEventUpdateTime(tableIf.getUpdateTime());
+ table.setUpdateTime(tableIf.getUpdateTime());
}
return true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 8e8556c8716..47ed5ff6a6a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -190,9 +190,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
private HMSDlaTable dlaTable;
- // record the event update time when enable hms event listener
- protected volatile long eventUpdateTime;
-
public enum DLAType {
UNKNOWN, HIVE, HUDI, ICEBERG
}
@@ -648,11 +645,11 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
public Optional<SchemaCacheValue> initSchemaAndUpdateTime(SchemaCacheKey
key) {
Table table = loadHiveTable();
// try to use transient_lastDdlTime from hms client
- schemaUpdateTime = MapUtils.isNotEmpty(table.getParameters())
+ setUpdateTime(MapUtils.isNotEmpty(table.getParameters())
&&
table.getParameters().containsKey(TBL_PROP_TRANSIENT_LAST_DDL_TIME)
?
Long.parseLong(table.getParameters().get(TBL_PROP_TRANSIENT_LAST_DDL_TIME)) *
1000
// use current timestamp if lastDdlTime does not exist (hive
views don't have this prop)
- : System.currentTimeMillis();
+ : System.currentTimeMillis());
return initSchema(key);
}
@@ -902,17 +899,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
builder.setMaxValue(Double.POSITIVE_INFINITY);
}
- public void setEventUpdateTime(long updateTime) {
- this.eventUpdateTime = updateTime;
- }
-
- @Override
- // get the max value of `schemaUpdateTime` and `eventUpdateTime`
- // eventUpdateTime will be refreshed after processing events with hms
event listener enabled
- public long getUpdateTime() {
- return Math.max(this.schemaUpdateTime, this.eventUpdateTime);
- }
-
@Override
public void gsonPostProcess() throws IOException {
super.gsonPostProcess();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index b1ede3926be..7d8501f449f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -337,7 +337,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
}
@Override
- public void afterTruncateTable(String dbName, String tblName) {
+ public void afterTruncateTable(String dbName, String tblName, long
updateTime) {
try {
// Invalidate cache.
Optional<ExternalDatabase<?>> db = catalog.getDbForReplay(dbName);
@@ -345,7 +345,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
Optional tbl = db.get().getTableForReplay(tblName);
if (tbl.isPresent()) {
Env.getCurrentEnv().getRefreshManager()
- .refreshTableInternal(db.get(), (ExternalTable)
tbl.get(), 0);
+ .refreshTableInternal(db.get(), (ExternalTable)
tbl.get(), updateTime);
}
}
} catch (Exception e) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index 25bf959bdd5..76cd2776ab2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -231,46 +231,48 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
/**
* Add partition field to Iceberg table for partition evolution
*/
- public void addPartitionField(IcebergExternalTable table,
AddPartitionFieldClause clause) throws UserException {
+ public void addPartitionField(IcebergExternalTable table,
AddPartitionFieldClause clause, long updateTime)
+ throws UserException {
makeSureInitialized();
if (metadataOps == null) {
throw new UserException("Add partition field operation is not
supported for catalog: " + getName());
}
- ((IcebergMetadataOps) metadataOps).addPartitionField(table, clause);
+ ((IcebergMetadataOps) metadataOps).addPartitionField(table, clause,
updateTime);
Env.getCurrentEnv().getEditLog()
.logRefreshExternalTable(
ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(),
- table.getDbName(), table.getName()));
+ table.getDbName(), table.getName(),
updateTime));
}
/**
* Drop partition field from Iceberg table for partition evolution
*/
- public void dropPartitionField(IcebergExternalTable table,
DropPartitionFieldClause clause) throws UserException {
+ public void dropPartitionField(IcebergExternalTable table,
DropPartitionFieldClause clause, long updateTime)
+ throws UserException {
makeSureInitialized();
if (metadataOps == null) {
throw new UserException("Drop partition field operation is not
supported for catalog: " + getName());
}
- ((IcebergMetadataOps) metadataOps).dropPartitionField(table, clause);
+ ((IcebergMetadataOps) metadataOps).dropPartitionField(table, clause,
updateTime);
Env.getCurrentEnv().getEditLog()
.logRefreshExternalTable(
ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(),
- table.getDbName(), table.getName()));
+ table.getDbName(), table.getName(),
updateTime));
}
/**
* Replace partition field in Iceberg table for partition evolution
*/
public void replacePartitionField(IcebergExternalTable table,
- ReplacePartitionFieldClause clause) throws UserException {
+ ReplacePartitionFieldClause clause, long updateTime) throws
UserException {
makeSureInitialized();
if (metadataOps == null) {
throw new UserException("Replace partition field operation is not
supported for catalog: " + getName());
}
- ((IcebergMetadataOps) metadataOps).replacePartitionField(table,
clause);
+ ((IcebergMetadataOps) metadataOps).replacePartitionField(table,
clause, updateTime);
Env.getCurrentEnv().getEditLog()
.logRefreshExternalTable(
ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(),
- table.getDbName(), table.getName()));
+ table.getDbName(), table.getName(),
updateTime));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index c52c5068003..c2795a76706 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -623,19 +623,19 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
- private void refreshTable(ExternalTable dorisTable) {
+ private void refreshTable(ExternalTable dorisTable, long updateTime) {
Optional<ExternalDatabase<?>> db =
dorisCatalog.getDbForReplay(dorisTable.getRemoteDbName());
if (db.isPresent()) {
Optional<?> tbl =
db.get().getTableForReplay(dorisTable.getRemoteName());
if (tbl.isPresent()) {
Env.getCurrentEnv().getRefreshManager()
- .refreshTableInternal(db.get(), (ExternalTable)
tbl.get(), System.currentTimeMillis());
+ .refreshTableInternal(db.get(), (ExternalTable)
tbl.get(), updateTime);
}
}
}
@Override
- public void addColumn(ExternalTable dorisTable, Column column,
ColumnPosition position)
+ public void addColumn(ExternalTable dorisTable, Column column,
ColumnPosition position, long updateTime)
throws UserException {
validateCommonColumnInfo(column);
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
@@ -650,11 +650,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to add column: " +
column.getName() + " to table: "
+ icebergTable.name() + ", error message is: " +
e.getMessage(), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
}
@Override
- public void addColumns(ExternalTable dorisTable, List<Column> columns)
throws UserException {
+ public void addColumns(ExternalTable dorisTable, List<Column> columns,
long updateTime) throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
UpdateSchema updateSchema = icebergTable.updateSchema();
for (Column column : columns) {
@@ -667,11 +667,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to add columns to table: " +
icebergTable.name()
+ ", error message is: " + e.getMessage(), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
}
@Override
- public void dropColumn(ExternalTable dorisTable, String columnName) throws
UserException {
+ public void dropColumn(ExternalTable dorisTable, String columnName, long
updateTime) throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
UpdateSchema updateSchema = icebergTable.updateSchema();
updateSchema.deleteColumn(columnName);
@@ -681,11 +681,12 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to drop column: " + columnName + "
from table: "
+ icebergTable.name() + ", error message is: " +
e.getMessage(), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
}
@Override
- public void renameColumn(ExternalTable dorisTable, String oldName, String
newName) throws UserException {
+ public void renameColumn(ExternalTable dorisTable, String oldName, String
newName, long updateTime)
+ throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
UpdateSchema updateSchema = icebergTable.updateSchema();
updateSchema.renameColumn(oldName, newName);
@@ -695,11 +696,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to rename column: " + oldName + "
to " + newName
+ " in table: " + icebergTable.name() + ", error message
is: " + e.getMessage(), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
}
@Override
- public void modifyColumn(ExternalTable dorisTable, Column column,
ColumnPosition position)
+ public void modifyColumn(ExternalTable dorisTable, Column column,
ColumnPosition position, long updateTime)
throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
validateForModifyColumn(column, icebergTable);
@@ -720,7 +721,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to modify column: " +
column.getName() + " in table: "
+ icebergTable.name() + ", error message is: " +
e.getMessage(), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
}
private void validateForModifyColumn(Column column, Table icebergTable)
throws UserException {
@@ -752,7 +753,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
@Override
- public void reorderColumns(ExternalTable dorisTable, List<String>
newOrder) throws UserException {
+ public void reorderColumns(ExternalTable dorisTable, List<String>
newOrder, long updateTime) throws UserException {
if (newOrder == null || newOrder.isEmpty()) {
throw new UserException("Reorder column failed, new order is
empty.");
}
@@ -768,7 +769,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to reorder columns in table: " +
icebergTable.name()
+ ", error message is: " + e.getMessage(), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
}
public ExecutionAuthenticator getExecutionAuthenticator() {
@@ -810,7 +811,8 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
/**
* Add partition field to Iceberg table for partition evolution
*/
- public void addPartitionField(ExternalTable dorisTable,
AddPartitionFieldClause clause) throws UserException {
+ public void addPartitionField(ExternalTable dorisTable,
AddPartitionFieldClause clause, long updateTime)
+ throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
UpdatePartitionSpec updateSpec = icebergTable.updateSpec();
@@ -832,7 +834,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to add partition field to table: "
+ icebergTable.name()
+ ", error message is: " +
ExceptionUtils.getRootCauseMessage(e), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
// Reset cached isValidRelatedTable flag after partition evolution
((IcebergExternalTable)
dorisTable).setIsValidRelatedTableCached(false);
}
@@ -840,7 +842,8 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
/**
* Drop partition field from Iceberg table for partition evolution
*/
- public void dropPartitionField(ExternalTable dorisTable,
DropPartitionFieldClause clause) throws UserException {
+ public void dropPartitionField(ExternalTable dorisTable,
DropPartitionFieldClause clause, long updateTime)
+ throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
UpdatePartitionSpec updateSpec = icebergTable.updateSpec();
@@ -860,7 +863,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to drop partition field from
table: " + icebergTable.name()
+ ", error message is: " +
ExceptionUtils.getRootCauseMessage(e), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
// Reset cached isValidRelatedTable flag after partition evolution
((IcebergExternalTable)
dorisTable).setIsValidRelatedTableCached(false);
}
@@ -868,7 +871,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
/**
* Replace partition field in Iceberg table for partition evolution
*/
- public void replacePartitionField(ExternalTable dorisTable,
ReplacePartitionFieldClause clause)
+ public void replacePartitionField(ExternalTable dorisTable,
ReplacePartitionFieldClause clause, long updateTime)
throws UserException {
Table icebergTable = IcebergUtils.getIcebergTable(dorisTable);
UpdatePartitionSpec updateSpec = icebergTable.updateSpec();
@@ -903,7 +906,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
throw new UserException("Failed to replace partition field in
table: " + icebergTable.name()
+ ", error message is: " +
ExceptionUtils.getRootCauseMessage(e), e);
}
- refreshTable(dorisTable);
+ refreshTable(dorisTable, updateTime);
// Reset cached isValidRelatedTable flag after partition evolution
((IcebergExternalTable)
dorisTable).setIsValidRelatedTableCached(false);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
index 35fcea59048..3407e8e7cec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
@@ -140,14 +140,14 @@ public interface ExternalMetadataOps {
* @param dorisTable
* @param partitions
*/
- default void truncateTable(ExternalTable dorisTable, List<String>
partitions) throws DdlException {
+ default void truncateTable(ExternalTable dorisTable, List<String>
partitions, long updateTime) throws DdlException {
truncateTableImpl(dorisTable, partitions);
- afterTruncateTable(dorisTable.getDbName(), dorisTable.getName());
+ afterTruncateTable(dorisTable.getDbName(), dorisTable.getName(),
updateTime);
}
void truncateTableImpl(ExternalTable dorisTable, List<String> partitions)
throws DdlException;
- default void afterTruncateTable(String dbName, String tblName) {
+ default void afterTruncateTable(String dbName, String tblName, long
updateTime) {
}
/**
@@ -223,7 +223,7 @@ public interface ExternalMetadataOps {
* @param position
* @throws UserException
*/
- default void addColumn(ExternalTable dorisTable, Column column,
ColumnPosition position)
+ default void addColumn(ExternalTable dorisTable, Column column,
ColumnPosition position, long updateTime)
throws UserException {
throw new UnsupportedOperationException("Add column operation is not
supported for this table type.");
}
@@ -235,7 +235,7 @@ public interface ExternalMetadataOps {
* @param columns
* @throws UserException
*/
- default void addColumns(ExternalTable dorisTable, List<Column> columns)
+ default void addColumns(ExternalTable dorisTable, List<Column> columns,
long updateTime)
throws UserException {
throw new UnsupportedOperationException("Add columns operation is not
supported for this table type.");
}
@@ -247,7 +247,7 @@ public interface ExternalMetadataOps {
* @param columnName
* @throws UserException
*/
- default void dropColumn(ExternalTable dorisTable, String columnName)
+ default void dropColumn(ExternalTable dorisTable, String columnName, long
updateTime)
throws UserException {
throw new UnsupportedOperationException("Drop column operation is not
supported for this table type.");
}
@@ -260,7 +260,7 @@ public interface ExternalMetadataOps {
* @param newName
* @throws UserException
*/
- default void renameColumn(ExternalTable dorisTable, String oldName, String
newName)
+ default void renameColumn(ExternalTable dorisTable, String oldName, String
newName, long updateTime)
throws UserException {
throw new UnsupportedOperationException("Rename column operation is
not supported for this table type.");
}
@@ -273,7 +273,7 @@ public interface ExternalMetadataOps {
* @param position
* @throws UserException
*/
- default void modifyColumn(ExternalTable dorisTable, Column column,
ColumnPosition position)
+ default void modifyColumn(ExternalTable dorisTable, Column column,
ColumnPosition position, long updateTime)
throws UserException {
throw new UnsupportedOperationException("Modify column operation is
not supported for this table type.");
}
@@ -285,7 +285,7 @@ public interface ExternalMetadataOps {
* @param newOrder
* @throws UserException
*/
- default void reorderColumns(ExternalTable dorisTable, List<String>
newOrder)
+ default void reorderColumns(ExternalTable dorisTable, List<String>
newOrder, long updateTime)
throws UserException {
throw new UnsupportedOperationException("Reorder columns operation is
not supported for this table type.");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java
index 6d08b10403b..d3277fe0756 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java
@@ -41,6 +41,7 @@ public class TestExternalTable extends ExternalTable {
public synchronized void makeSureInitialized() {
super.makeSureInitialized();
this.objectCreated = true;
+ setUpdateTime(System.currentTimeMillis());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java
index e23d63b3aaa..394b4c25751 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java
@@ -105,7 +105,7 @@ public class ExecuteActionCommand extends Command
implements ForwardWithSync {
action.validate(tableNameInfo, ctx.getCurrentUserIdentity());
ResultSet resultSet = action.execute(table);
- logRefreshTable(table);
+ logRefreshTable(table, System.currentTimeMillis());
if (resultSet != null) {
executor.sendResultSet(resultSet);
}
@@ -150,7 +150,7 @@ public class ExecuteActionCommand extends Command
implements ForwardWithSync {
* @param table the table to log
* @throws UserException if the table type is not supported
*/
- private void logRefreshTable(TableIf table) throws UserException {
+ private void logRefreshTable(TableIf table, long updateTime) throws
UserException {
if (table instanceof ExternalTable) {
ExternalTable externalTable = (ExternalTable) table;
Env.getCurrentEnv().getEditLog()
@@ -158,7 +158,7 @@ public class ExecuteActionCommand extends Command
implements ForwardWithSync {
ExternalObjectLog.createForRefreshTable(
externalTable.getCatalog().getId(),
externalTable.getDbName(),
- externalTable.getName()));
+ externalTable.getName(), updateTime));
} else {
// support more table in future
throw new UserException("Unsupported table type: " +
table.getClass().getName() + " for refresh table");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
index 9f8be52afec..fef4d283165 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
@@ -98,6 +98,8 @@ public class HiveInsertExecutor extends
BaseExternalTableInsertExecutor {
}
// Write edit log to notify other FEs
+ long updateTime = System.currentTimeMillis();
+ hmsTable.setUpdateTime(updateTime);
ExternalObjectLog log;
if (!modifiedPartNames.isEmpty() || !newPartNames.isEmpty()) {
// Partition-level refresh for other FEs
@@ -106,13 +108,14 @@ public class HiveInsertExecutor extends
BaseExternalTableInsertExecutor {
table.getDatabase().getFullName(),
table.getName(),
modifiedPartNames,
- newPartNames);
+ newPartNames,
+ updateTime);
} else {
// Full table refresh for other FEs
log = ExternalObjectLog.createForRefreshTable(
hmsTable.getCatalog().getId(),
table.getDatabase().getFullName(),
- table.getName());
+ table.getName(), updateTime);
}
Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
index b9d77ca6818..b846d1acbdc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
@@ -58,6 +58,8 @@ public class TruncateTableInfo implements Writable {
private boolean force = true; // older version it was forced always.
@SerializedName(value = "ur")
private Map<Long, Long> updateRecords;
+ @SerializedName(value = "ut")
+ private long updateTime;
public TruncateTableInfo() {
@@ -82,11 +84,12 @@ public class TruncateTableInfo implements Writable {
}
// for external table
- public TruncateTableInfo(String ctl, String db, String table, List<String>
partNames) {
+ public TruncateTableInfo(String ctl, String db, String table, List<String>
partNames, long updateTime) {
this.ctl = ctl;
this.db = db;
this.table = table;
this.extPartNames = partNames;
+ this.updateTime = updateTime;
}
public String getCtl() {
@@ -137,6 +140,10 @@ public class TruncateTableInfo implements Writable {
return updateRecords;
}
+ public long getUpdateTime() {
+ return updateTime;
+ }
+
public static TruncateTableInfo read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, TruncateTableInfo.class);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
index 32703c682a3..c9dd169bc78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
@@ -136,7 +136,7 @@ public class CacheAnalyzer {
enableSqlCache = true;
}
}
- // alread remove the entrance of partition cache, so we force set to
false
+ // already remove the entrance of partition cache, so we force set to
false
enablePartitionCache = false;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
index 0a1cd323d8f..a5f2465c6e7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
@@ -88,12 +88,12 @@ public class RefreshTableTest extends TestWithFeService {
CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
Assertions.assertFalse(table.isObjectCreated());
- long l1 = table.getSchemaUpdateTime();
- Assertions.assertTrue(l1 == 0);
+ long l1 = table.getUpdateTime();
+ // Assertions.assertEquals(0, l1);
table.makeSureInitialized();
Assertions.assertTrue(table.isObjectCreated());
- long l2 = table.getSchemaUpdateTime();
- Assertions.assertTrue(l2 == l1);
+ long l2 = table.getUpdateTime();
+ Assertions.assertTrue(l2 >= l1);
TableNameInfo tableNameInfo = new TableNameInfo("test1", "db1",
"tbl11");
try {
Env.getCurrentEnv().getRefreshManager()
@@ -102,15 +102,15 @@ public class RefreshTableTest extends TestWithFeService {
// Do nothing
}
Assertions.assertFalse(table.isObjectCreated());
- long l3 = table.getSchemaUpdateTime();
- Assertions.assertTrue(l3 == l2);
+ long l3 = table.getUpdateTime();
+ Assertions.assertTrue(l3 >= l2);
table.getFullSchema();
// only table.getFullSchema() can change table.lastUpdateTime
- long l4 = table.getSchemaUpdateTime();
+ long l4 = table.getUpdateTime();
Assertions.assertTrue(l4 > l3);
// updateTime is equal to schema update time as default
long l5 = table.getUpdateTime();
- Assertions.assertTrue(l5 == l4);
+ Assertions.assertTrue(l5 >= l4);
// external info schema db
ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase)
test1.getDbNullable(InfoSchemaDb.DATABASE_NAME);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
index 7d358073ab8..43d5669e6ad 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java
@@ -106,8 +106,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
Deencapsulation.setField(db, "initialized", true);
Deencapsulation.setField(tbl, "objectCreated", true);
- Deencapsulation.setField(tbl, "schemaUpdateTime", NOW);
- Deencapsulation.setField(tbl, "eventUpdateTime", 0);
+ Deencapsulation.setField(tbl, "updateTime", NOW);
Deencapsulation.setField(tbl, "catalog", hmsCatalog);
Deencapsulation.setField(tbl, "dbName", "hms_db");
Deencapsulation.setField(tbl, "name", "hms_tbl");
@@ -163,8 +162,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view1, "objectCreated", true);
- Deencapsulation.setField(view1, "schemaUpdateTime", NOW);
- Deencapsulation.setField(view1, "eventUpdateTime", 0);
+ Deencapsulation.setField(view1, "updateTime", NOW);
Deencapsulation.setField(view1, "catalog", hmsCatalog);
Deencapsulation.setField(view1, "dbName", "hms_db");
Deencapsulation.setField(view1, "name", "hms_view1");
@@ -215,8 +213,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view2, "objectCreated", true);
- Deencapsulation.setField(view2, "schemaUpdateTime", NOW);
- Deencapsulation.setField(view2, "eventUpdateTime", 0);
+ Deencapsulation.setField(view2, "updateTime", NOW);
Deencapsulation.setField(view2, "catalog", hmsCatalog);
Deencapsulation.setField(view2, "dbName", "hms_db");
Deencapsulation.setField(view2, "name", "hms_view2");
@@ -268,8 +265,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view3, "objectCreated", true);
- Deencapsulation.setField(view3, "schemaUpdateTime", NOW);
- Deencapsulation.setField(view3, "eventUpdateTime", 0);
+ Deencapsulation.setField(view3, "updateTime", NOW);
Deencapsulation.setField(view3, "catalog", hmsCatalog);
Deencapsulation.setField(view3, "dbName", "hms_db");
Deencapsulation.setField(view3, "name", "hms_view3");
@@ -321,8 +317,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view4, "objectCreated", true);
- Deencapsulation.setField(view4, "schemaUpdateTime", NOW);
- Deencapsulation.setField(view4, "eventUpdateTime", 0);
+ Deencapsulation.setField(view4, "updateTime", NOW);
Deencapsulation.setField(view4, "catalog", hmsCatalog);
Deencapsulation.setField(view4, "dbName", "hms_db");
Deencapsulation.setField(view4, "name", "hms_view4");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
index bbca8c183f1..2e69f7e31f9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
@@ -136,8 +136,7 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase
{
setField(db, "initialized", true);
setField(tbl, "objectCreated", true);
- setField(tbl, "schemaUpdateTime", NOW);
- setField(tbl, "eventUpdateTime", 0);
+ setField(tbl, "updateTime", NOW);
setField(tbl, "catalog", hmsCatalog);
setField(tbl, "dbName", "hms_db");
setField(tbl, "name", "hms_tbl");
@@ -159,8 +158,7 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase
{
.thenReturn(Optional.empty());
setField(tbl2, "objectCreated", true);
- setField(tbl2, "schemaUpdateTime", NOW);
- setField(tbl2, "eventUpdateTime", 0);
+ setField(tbl2, "updateTime", NOW);
setField(tbl2, "catalog", hmsCatalog);
setField(tbl2, "dbName", "hms_db");
setField(tbl2, "name", "hms_tbl2");
@@ -178,11 +176,11 @@ public class HmsQueryCacheTest extends
AnalyzeCheckTestBase {
Mockito.when(tbl2.getDatabase()).thenReturn(db);
Mockito.when(tbl2.getSupportedSysTables()).thenReturn(SupportedSysTables.HIVE_SUPPORTED_SYS_TABLES);
Mockito.when(tbl2.getUpdateTime()).thenReturn(NOW);
- Mockito.when(tbl2.getSchemaUpdateTime()).thenReturn(NOW);
+ Mockito.when(tbl2.getUpdateTime()).thenReturn(NOW);
// mock initSchemaAndUpdateTime and do nothing
Mockito.when(tbl2.initSchemaAndUpdateTime(Mockito.any(ExternalSchemaCache.SchemaCacheKey.class)))
.thenReturn(Optional.empty());
- Mockito.doNothing().when(tbl2).setEventUpdateTime(Mockito.anyLong());
+ Mockito.doNothing().when(tbl2).setUpdateTime(Mockito.anyLong());
setField(view1, "objectCreated", true);
@@ -259,7 +257,7 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase
{
SqlCache sqlCache1 = (SqlCache) ca.getCache();
// latestTime is equals to the schema update time if not set partition
update time
- Assert.assertEquals(tbl2.getSchemaUpdateTime(),
sqlCache1.getLatestTime());
+ Assert.assertEquals(tbl2.getUpdateTime(), sqlCache1.getLatestTime());
// wait a second and set partition update time
try {
diff --git
a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy
b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy
index f5d38350e7a..b8838eadbaa 100644
---
a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy
@@ -53,6 +53,9 @@ suite("iceberg_branch_tag_operate",
"p0,external,doris,external_docker,external_
sql """ alter table test_branch_tag_operate create branch b1 """
sql """ alter table test_branch_tag_operate create branch if not exists b1
"""
+ def result = sql """select UPDATE_TIME from information_schema.tables
where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'"""
+ def update_time1 = result[0][0];
+ sleep(1000)
test {
sql """ alter table test_branch_tag_operate create or replace branch
b1 """
@@ -72,7 +75,11 @@ suite("iceberg_branch_tag_operate",
"p0,external,doris,external_docker,external_
sql """ insert into test_branch_tag_operate values (3) """
sql """ insert into test_branch_tag_operate values (4) """
sql """ insert into test_branch_tag_operate values (5) """
-
+ result = sql """select UPDATE_TIME from information_schema.tables where
TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'"""
+ def update_time2 = result[0][0];
+ logger.info("get update times " + update_time1 + " vs. " + update_time2)
+ assertTrue(update_time2 > update_time1);
+ sleep(1000)
List<List<Object>> snapshots = sql """ select snapshot_id from
iceberg_meta("table" = "${catalog_name}.test_db.test_branch_tag_operate",
"query_type" = "snapshots") order by committed_at; """
String s0 = snapshots.get(0)[0]
@@ -137,6 +144,10 @@ suite("iceberg_branch_tag_operate",
"p0,external,doris,external_docker,external_
exception "Cannot set b8 to unknown snapshot: 11223344"
}
+ result = sql """select UPDATE_TIME from information_schema.tables where
TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'"""
+ def update_time3 = result[0][0];
+ logger.info("get update times " + update_time2 + " vs. " + update_time3)
+ assertTrue(update_time3 > update_time2);
// tag
sql """ alter table test_branch_tag_operate create tag t2 as of version
${s0} """
diff --git
a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
index bc6002500ea..33ce0648110 100644
---
a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy
@@ -73,6 +73,9 @@ suite("iceberg_schema_change_ddl",
"p0,external,doris,external_docker,external_d
(2, 'Bob', 30, 87.2),
(3, 'Charlie', 22, 92.8)
"""
+ def result = sql """select UPDATE_TIME from information_schema.tables
where TABLE_SCHEMA="iceberg_schema_change_ddl_db" and
TABLE_NAME='${table_name}'"""
+ def update_time1 = result[0][0];
+ sleep(1000)
// Verify initial state
qt_init_1 """ DESC ${table_name} """
@@ -104,6 +107,10 @@ suite("iceberg_schema_change_ddl",
"p0,external,doris,external_docker,external_d
// Test 3: ADD complex type column
sql """ ALTER TABLE ${table_name} ADD COLUMN address STRUCT<city: STRING,
country: STRING> """
+ result = sql """select UPDATE_TIME from information_schema.tables where
TABLE_SCHEMA="iceberg_schema_change_ddl_db" and TABLE_NAME='${table_name}'"""
+ def update_time2 = result[0][0];
+ logger.info("get update times " + update_time1 + " vs. " + update_time2)
+ assertTrue(update_time2 > update_time1);
qt_add_multi_1 """ DESC ${table_name} """
diff --git
a/regression-test/suites/external_table_p0/test_external_table_update_time.groovy
b/regression-test/suites/external_table_p0/test_external_table_update_time.groovy
new file mode 100644
index 00000000000..9d18db8aa03
--- /dev/null
+++
b/regression-test/suites/external_table_p0/test_external_table_update_time.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_external_table_update_time",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Hive test.")
+ return
+ }
+
+ for (String hivePrefix : ["hive3"]) {
+ String extHiveHmsHost =
context.config.otherConfigs.get("externalEnvIp")
+ String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String catalog_name = hivePrefix + "_test_update_time_ctl"
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+
+ sql "drop database if exists test_update_time_db force";
+ sql "create database test_update_time_db";
+ sql "use test_update_time_db";
+ sql "create table test_update_time_tbl(k1 int)"
+ sql "insert into test_update_time_tbl values(1)"
+ def result = sql """select UPDATE_TIME from information_schema.tables
where TABLE_SCHEMA="test_update_time_db" and
TABLE_NAME="test_update_time_tbl""""
+ def update_time1 = result[0][0];
+ sleep(2000);
+ sql "insert into test_update_time_tbl values(2)"
+ result = sql """select UPDATE_TIME from information_schema.tables
where TABLE_SCHEMA="test_update_time_db" and
TABLE_NAME="test_update_time_tbl""""
+ def update_time2 = result[0][0];
+ logger.info("get update times " + update_time1 + " vs. " +
update_time2)
+ assertTrue(update_time2 > update_time1);
+ sleep(2000);
+ sql "truncate table test_update_time_tbl";
+ result = sql """select UPDATE_TIME from information_schema.tables
where TABLE_SCHEMA="test_update_time_db" and
TABLE_NAME="test_update_time_tbl""""
+ def update_time3 = result[0][0];
+ logger.info("get update times " + update_time2 + " vs. " +
update_time3)
+ assertTrue(update_time3 > update_time2);
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
