This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8530065402c9d91aa79d4dec58a3ca4602ad377c
Author: zhangdong <493738...@qq.com>
AuthorDate: Fri Jul 21 09:42:35 2023 +0800
[improvement](multi-catalog)add last sync time for external catalog (#21873)
which operation can update this time:
1.when refresh catalog,lastUpdateTime of catalog will be update
2.when refresh db,lastUpdateTime of db will be update
3.when reload table schema to cache,lastUpdateTime of dbtable will be update
4.when receive add/drop table event,lastUpdateTime of db will be update
5.when receive alter table event,lastUpdateTime of table will be update
---
.../org/apache/doris/analysis/ShowCatalogStmt.java | 1 +
.../java/org/apache/doris/catalog/DatabaseIf.java | 4 ++
.../java/org/apache/doris/catalog/TableIf.java | 4 ++
.../doris/catalog/external/ExternalDatabase.java | 15 +++++
.../doris/catalog/external/ExternalTable.java | 7 ++
.../apache/doris/common/proc/CatalogsProcDir.java | 4 +-
.../org/apache/doris/common/proc/DbsProcDir.java | 4 +-
.../apache/doris/common/proc/TablesProcDir.java | 4 +-
.../org/apache/doris/datasource/CatalogIf.java | 4 ++
.../org/apache/doris/datasource/CatalogMgr.java | 7 +-
.../apache/doris/datasource/ExternalCatalog.java | 16 ++++-
.../apache/doris/datasource/ExternalObjectLog.java | 3 +
.../apache/doris/datasource/InitCatalogLog.java | 3 +
.../apache/doris/datasource/InitDatabaseLog.java | 3 +
.../apache/doris/analysis/ShowCatalogStmtTest.java | 6 +-
...freshTableTest.java => RefreshCatalogTest.java} | 65 ++++--------------
.../{RefreshTableTest.java => RefreshDbTest.java} | 67 ++++---------------
.../org/apache/doris/catalog/RefreshTableTest.java | 78 +++-------------------
.../apache/doris/common/proc/DbsProcDirTest.java | 12 ++--
19 files changed, 117 insertions(+), 190 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
index 8c32bafe5f..191c5ba9a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
@@ -32,6 +32,7 @@ public class ShowCatalogStmt extends ShowStmt {
.addColumn(new Column("Type",
ScalarType.createStringType()))
.addColumn(new Column("IsCurrent",
ScalarType.createStringType()))
.addColumn(new Column("CreateTime",
ScalarType.createStringType()))
+ .addColumn(new Column("LastUpdateTime",
ScalarType.createStringType()))
.addColumn(new Column("Comment",
ScalarType.createStringType()))
.build();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
index 7672b8b76d..f9c5d94601 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
@@ -235,4 +235,8 @@ public interface DatabaseIf<T extends TableIf> {
void dropTable(String tableName);
CatalogIf getCatalog();
+
+ default long getLastUpdateTime() {
+ return -1L;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index e9d0e91c25..78717f0eca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -232,5 +232,9 @@ public interface TableIf {
default boolean isManagedTable() {
return getType() == TableType.OLAP || getType() ==
TableType.MATERIALIZED_VIEW;
}
+
+ default long getLastUpdateTime() {
+ return -1L;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
index e1b582a540..0a82d37ff3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
@@ -73,6 +73,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
protected Map<String, Long> tableNameToId = Maps.newConcurrentMap();
@SerializedName(value = "idToTbl")
protected Map<Long, T> idToTbl = Maps.newConcurrentMap();
+ @SerializedName(value = "lastUpdateTime")
+ protected long lastUpdateTime;
protected final InitDatabaseLog.Type dbLogType;
protected ExternalCatalog extCatalog;
protected boolean invalidCacheInInit = true;
@@ -147,6 +149,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
tableNameToId = tmpTableNameToId;
idToTbl = tmpIdToTbl;
+ lastUpdateTime = log.getLastUpdateTime();
initialized = true;
}
@@ -179,6 +182,10 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
tableNameToId = tmpTableNameToId;
idToTbl = tmpIdToTbl;
}
+
+ long currentTime = System.currentTimeMillis();
+ lastUpdateTime = currentTime;
+ initDatabaseLog.setLastUpdateTime(lastUpdateTime);
initialized = true;
Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog);
}
@@ -307,6 +314,14 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
return idToTbl.get(tableId);
}
+ public long getLastUpdateTime() {
+ return lastUpdateTime;
+ }
+
+ public void setLastUpdateTime(long lastUpdateTime) {
+ this.lastUpdateTime = lastUpdateTime;
+ }
+
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, GsonUtils.GSON.toJson(this));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
index 88ba9bc94e..f4c76cda7a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
@@ -69,6 +69,8 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
protected long timestamp;
@SerializedName(value = "dbName")
protected String dbName;
+ @SerializedName(value = "lastUpdateTime")
+ protected long lastUpdateTime;
protected boolean objectCreated;
protected ExternalCatalog catalog;
@@ -342,6 +344,11 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
*
* @return
*/
+ public List<Column> initSchemaAndUpdateTime() {
+ lastUpdateTime = System.currentTimeMillis();
+ return initSchema();
+ }
+
public List<Column> initSchema() {
throw new NotImplementedException("implement in sub class");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
index 48e6b6ab46..854b4dddc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
@@ -20,6 +20,7 @@ package org.apache.doris.common.proc;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.ListComparator;
+import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.CatalogIf;
import com.google.common.base.Preconditions;
@@ -37,7 +38,7 @@ import java.util.List;
*/
public class CatalogsProcDir implements ProcDirInterface {
public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
- .add("CatalogIds").add("CatalogName").add("DatabaseNum")
+
.add("CatalogIds").add("CatalogName").add("DatabaseNum").add("LastUpdateTime")
.build();
private Env env;
@@ -90,6 +91,7 @@ public class CatalogsProcDir implements ProcDirInterface {
catalogInfo.add(catalog.getId());
catalogInfo.add(catalog.getName());
catalogInfo.add(catalog.getDbNames().size());
+
catalogInfo.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
catalogInfos.add(catalogInfo);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
index 431c618a6f..a3818099d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
@@ -43,7 +43,7 @@ public class DbsProcDir implements ProcDirInterface {
public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
.add("DbId").add("DbName").add("TableNum").add("Size").add("Quota")
.add("LastConsistencyCheckTime").add("ReplicaCount").add("ReplicaQuota")
- .add("TransactionQuota")
+ .add("TransactionQuota").add("LastUpdateTime")
.build();
private Env env;
@@ -122,7 +122,7 @@ public class DbsProcDir implements ProcDirInterface {
dbInfo.add(replicaCount);
dbInfo.add(replicaQuota);
dbInfo.add(transactionQuota);
-
+ dbInfo.add(TimeUtils.longToTimeString(db.getLastUpdateTime()));
} finally {
db.readUnlock();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
index 3b43ec8871..be51b157de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
@@ -45,6 +45,7 @@ public class TablesProcDir implements ProcDirInterface {
public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
.add("TableId").add("TableName").add("IndexNum").add("PartitionColumnName")
.add("PartitionNum").add("State").add("Type").add("LastConsistencyCheckTime").add("ReplicaCount")
+ .add("LastUpdateTime")
.build();
private DatabaseIf db;
@@ -128,8 +129,7 @@ public class TablesProcDir implements ProcDirInterface {
tableInfo.add(FeConstants.null_string);
tableInfo.add(replicaCount);
}
-
-
+
tableInfo.add(TimeUtils.longToTimeString(table.getLastUpdateTime()));
tableInfos.add(tableInfo);
} finally {
table.readUnlock();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index ce13c9881e..8e3304e54a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -152,6 +152,10 @@ public interface CatalogIf<T extends DatabaseIf> {
String getComment();
+ default long getLastUpdateTime() {
+ return -1L;
+ }
+
default CatalogLog constructEditLog() {
CatalogLog log = new CatalogLog();
log.setCatalogId(getId());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index a9a00fc61d..12fb7215f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -43,6 +43,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
@@ -376,7 +377,6 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
matcher =
PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(),
CaseSensibility.CATALOG.getCaseSensibility());
}
-
for (CatalogIf catalog : nameToCatalog.values()) {
if (Env.getCurrentEnv().getAccessManager()
.checkCtlPriv(ConnectContext.get(),
catalog.getName(), PrivPredicate.SHOW)) {
@@ -397,6 +397,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
Map<String, String> props = catalog.getProperties();
String createTime =
props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED");
row.add(createTime);
+
row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
row.add(catalog.getComment());
rows.add(row);
}
@@ -712,6 +713,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
log.setCatalogId(catalog.getId());
log.setDbId(db.getId());
log.setTableId(table.getId());
+ log.setLastUpdateTime(System.currentTimeMillis());
replayDropExternalTable(log);
Env.getCurrentEnv().getEditLog().logDropExternalTable(log);
}
@@ -737,6 +739,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
db.writeLock();
try {
db.dropTable(table.getName());
+ db.setLastUpdateTime(log.getLastUpdateTime());
} finally {
db.writeUnlock();
}
@@ -786,6 +789,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
log.setDbId(db.getId());
log.setTableName(tableName);
log.setTableId(Env.getCurrentEnv().getNextId());
+ log.setLastUpdateTime(System.currentTimeMillis());
replayCreateExternalTableFromEvent(log);
Env.getCurrentEnv().getEditLog().logCreateExternalTable(log);
}
@@ -806,6 +810,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
db.writeLock();
try {
db.replayCreateTableFromEvent(log.getTableName(),
log.getTableId());
+ db.setLastUpdateTime(log.getLastUpdateTime());
} finally {
db.writeUnlock();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index c169ed2c96..20ac85f53f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -84,6 +84,8 @@ public abstract class ExternalCatalog
private boolean initialized = false;
@SerializedName(value = "idToDb")
protected Map<Long, ExternalDatabase<? extends ExternalTable>> idToDb =
Maps.newConcurrentMap();
+ @SerializedName(value = "lastUpdateTime")
+ protected long lastUpdateTime;
// db name does not contains "default_cluster"
protected Map<String, Long> dbNameToId = Maps.newConcurrentMap();
private boolean objectCreated = false;
@@ -257,6 +259,9 @@ public abstract class ExternalCatalog
}
dbNameToId = tmpDbNameToId;
idToDb = tmpIdToDb;
+ long currentTime = System.currentTimeMillis();
+ lastUpdateTime = currentTime;
+ initCatalogLog.setLastUpdateTime(lastUpdateTime);
Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog);
}
@@ -279,7 +284,7 @@ public abstract class ExternalCatalog
if (db.isPresent()) {
Optional<? extends ExternalTable> table =
db.get().getTable(tblName);
if (table.isPresent()) {
- return table.get().initSchema();
+ return table.get().initSchemaAndUpdateTime();
}
}
// return one column with unsupported type.
@@ -403,6 +408,14 @@ public abstract class ExternalCatalog
props.remove("comment");
}
+ public long getLastUpdateTime() {
+ return lastUpdateTime;
+ }
+
+ public void setLastUpdateTime(long lastUpdateTime) {
+ this.lastUpdateTime = lastUpdateTime;
+ }
+
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -437,6 +450,7 @@ public abstract class ExternalCatalog
}
dbNameToId = tmpDbNameToId;
idToDb = tmpIdToDb;
+ lastUpdateTime = log.getLastUpdateTime();
initialized = true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
index 04d5b06036..4ebb137a60 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java
@@ -56,6 +56,9 @@ public class ExternalObjectLog implements Writable {
@SerializedName(value = "partitionNames")
private List<String> partitionNames;
+ @SerializedName(value = "lastUpdateTime")
+ private long lastUpdateTime;
+
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, GsonUtils.GSON.toJson(this));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
index 73fbeeb781..dd30fbf43c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
@@ -65,6 +65,9 @@ public class InitCatalogLog implements Writable {
@SerializedName(value = "type")
private Type type;
+ @SerializedName(value = "lastUpdateTime")
+ private long lastUpdateTime;
+
public InitCatalogLog() {
refreshCount = 0;
createCount = 0;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
index 14cd4410ec..3a85fb1edc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
@@ -68,6 +68,9 @@ public class InitDatabaseLog implements Writable {
@SerializedName(value = "type")
private Type type;
+ @SerializedName(value = "lastUpdateTime")
+ protected long lastUpdateTime;
+
public InitDatabaseLog() {
refreshCount = 0;
createCount = 0;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
index 68b7ae49ff..56e33f1b1f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
@@ -26,18 +26,18 @@ import org.junit.Test;
public class ShowCatalogStmtTest {
@Test
public void testNormal() throws UserException, AnalysisException {
- final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer();
+ final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer();
ShowCatalogStmt stmt = new ShowCatalogStmt();
stmt.analyze(analyzer);
Assert.assertNull(stmt.getCatalogName());
- Assert.assertEquals(6, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(7, stmt.getMetaData().getColumnCount());
Assert.assertEquals("SHOW CATALOGS", stmt.toSql());
stmt = new ShowCatalogStmt(null, "%hive%");
stmt.analyze(analyzer);
Assert.assertNull(stmt.getCatalogName());
Assert.assertNotNull(stmt.getPattern());
- Assert.assertEquals(6, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(7, stmt.getMetaData().getColumnCount());
Assert.assertEquals("SHOW CATALOGS LIKE '%hive%'", stmt.toSql());
stmt = new ShowCatalogStmt("testCatalog", null);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java
similarity index 70%
copy from
fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
copy to
fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java
index 11e37ce238..1a663d7a26 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java
@@ -20,10 +20,6 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.analysis.DropCatalogStmt;
import org.apache.doris.analysis.RefreshCatalogStmt;
-import org.apache.doris.analysis.RefreshDbStmt;
-import org.apache.doris.analysis.RefreshTableStmt;
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.TestExternalTable;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.CatalogIf;
@@ -41,7 +37,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
-public class RefreshTableTest extends TestWithFeService {
+public class RefreshCatalogTest extends TestWithFeService {
private static Env env;
private ConnectContext rootCtx;
@@ -71,7 +67,13 @@ public class RefreshTableTest extends TestWithFeService {
@Test
public void testRefreshCatalog() throws Exception {
CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
+ // init is 0
+ long l1 = test1.getLastUpdateTime();
+ Assertions.assertTrue(l1 == 0);
TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
+ // getDb() triggered init method
+ long l2 = test1.getLastUpdateTime();
+ Assertions.assertTrue(l2 > l1);
Assertions.assertFalse(table.isObjectCreated());
table.makeSureInitialized();
Assertions.assertTrue(table.isObjectCreated());
@@ -82,6 +84,9 @@ public class RefreshTableTest extends TestWithFeService {
} catch (Exception e) {
// Do nothing
}
+ // not triggered init method
+ long l3 = test1.getLastUpdateTime();
+ Assertions.assertTrue(l3 == l2);
Assertions.assertTrue(table.isObjectCreated());
test1.getDbNullable("db1").getTables();
Assertions.assertFalse(table.isObjectCreated());
@@ -93,6 +98,9 @@ public class RefreshTableTest extends TestWithFeService {
Assertions.assertFalse(((ExternalCatalog) test1).isInitialized());
table.makeSureInitialized();
Assertions.assertTrue(((ExternalCatalog) test1).isInitialized());
+ // table.makeSureInitialized() triggered init method
+ long l4 = test1.getLastUpdateTime();
+ Assertions.assertTrue(l4 > l3);
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
} catch (Exception e) {
@@ -101,53 +109,6 @@ public class RefreshTableTest extends TestWithFeService {
Assertions.assertFalse(((ExternalCatalog) test1).isInitialized());
}
- @Test
- public void testRefreshDatabase() throws Exception {
- CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
- Assertions.assertFalse(table.isObjectCreated());
- table.makeSureInitialized();
- Assertions.assertTrue(table.isObjectCreated());
- RefreshDbStmt refreshDbStmt = new RefreshDbStmt("test1", "db1", null);
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertTrue(table.isObjectCreated());
- test1.getDbNullable("db1").getTables();
- Assertions.assertFalse(table.isObjectCreated());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalDatabase)
test1.getDbNullable("db1")).isInitialized());
- table.makeSureInitialized();
- Assertions.assertTrue(((ExternalDatabase)
test1.getDbNullable("db1")).isInitialized());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
- } catch (Exception e) {
- // Do nothing
- }
- }
-
- @Test
- public void testRefreshTable() throws Exception {
- CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
- Assertions.assertFalse(table.isObjectCreated());
- table.makeSureInitialized();
- Assertions.assertTrue(table.isObjectCreated());
- RefreshTableStmt refreshTableStmt = new RefreshTableStmt(new
TableName("test1", "db1", "tbl11"));
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshTableStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(table.isObjectCreated());
- }
-
public static class RefreshTableProvider implements
TestExternalCatalog.TestCatalogProvider {
public static final Map<String, Map<String, List<Column>>> MOCKED_META;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
similarity index 67%
copy from
fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
copy to fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
index 11e37ce238..23def21689 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
@@ -19,15 +19,12 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.analysis.DropCatalogStmt;
-import org.apache.doris.analysis.RefreshCatalogStmt;
import org.apache.doris.analysis.RefreshDbStmt;
-import org.apache.doris.analysis.RefreshTableStmt;
-import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.external.ExternalDatabase;
+import org.apache.doris.catalog.external.TestExternalDatabase;
import org.apache.doris.catalog.external.TestExternalTable;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.DdlExecutor;
@@ -41,7 +38,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
-public class RefreshTableTest extends TestWithFeService {
+public class RefreshDbTest extends TestWithFeService {
private static Env env;
private ConnectContext rootCtx;
@@ -68,43 +65,15 @@ public class RefreshTableTest extends TestWithFeService {
env.getCatalogMgr().dropCatalog(stmt);
}
- @Test
- public void testRefreshCatalog() throws Exception {
- CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
- Assertions.assertFalse(table.isObjectCreated());
- table.makeSureInitialized();
- Assertions.assertTrue(table.isObjectCreated());
- RefreshCatalogStmt refreshCatalogStmt = new
RefreshCatalogStmt("test1", null);
- Assertions.assertTrue(refreshCatalogStmt.isInvalidCache());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertTrue(table.isObjectCreated());
- test1.getDbNullable("db1").getTables();
- Assertions.assertFalse(table.isObjectCreated());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalCatalog) test1).isInitialized());
- table.makeSureInitialized();
- Assertions.assertTrue(((ExternalCatalog) test1).isInitialized());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalCatalog) test1).isInitialized());
- }
-
@Test
public void testRefreshDatabase() throws Exception {
CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
+ TestExternalDatabase db1 = (TestExternalDatabase)
test1.getDbNullable("db1");
+ long l1 = db1.getLastUpdateTime();
+ Assertions.assertTrue(l1 == 0);
+ TestExternalTable table = db1.getTable("tbl11").get();
+ long l2 = db1.getLastUpdateTime();
+ Assertions.assertTrue(l2 > l1);
Assertions.assertFalse(table.isObjectCreated());
table.makeSureInitialized();
Assertions.assertTrue(table.isObjectCreated());
@@ -114,6 +83,8 @@ public class RefreshTableTest extends TestWithFeService {
} catch (Exception e) {
// Do nothing
}
+ long l3 = db1.getLastUpdateTime();
+ Assertions.assertTrue(l3 == l2);
Assertions.assertTrue(table.isObjectCreated());
test1.getDbNullable("db1").getTables();
Assertions.assertFalse(table.isObjectCreated());
@@ -124,6 +95,8 @@ public class RefreshTableTest extends TestWithFeService {
}
Assertions.assertFalse(((ExternalDatabase)
test1.getDbNullable("db1")).isInitialized());
table.makeSureInitialized();
+ long l4 = db1.getLastUpdateTime();
+ Assertions.assertTrue(l4 > l3);
Assertions.assertTrue(((ExternalDatabase)
test1.getDbNullable("db1")).isInitialized());
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
@@ -132,22 +105,6 @@ public class RefreshTableTest extends TestWithFeService {
}
}
- @Test
- public void testRefreshTable() throws Exception {
- CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
- Assertions.assertFalse(table.isObjectCreated());
- table.makeSureInitialized();
- Assertions.assertTrue(table.isObjectCreated());
- RefreshTableStmt refreshTableStmt = new RefreshTableStmt(new
TableName("test1", "db1", "tbl11"));
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshTableStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(table.isObjectCreated());
- }
-
public static class RefreshTableProvider implements
TestExternalCatalog.TestCatalogProvider {
public static final Map<String, Map<String, List<Column>>> MOCKED_META;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
index 11e37ce238..305bfa32c4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java
@@ -19,15 +19,11 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.analysis.DropCatalogStmt;
-import org.apache.doris.analysis.RefreshCatalogStmt;
-import org.apache.doris.analysis.RefreshDbStmt;
import org.apache.doris.analysis.RefreshTableStmt;
import org.apache.doris.analysis.TableName;
-import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.TestExternalTable;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.DdlExecutor;
@@ -68,77 +64,17 @@ public class RefreshTableTest extends TestWithFeService {
env.getCatalogMgr().dropCatalog(stmt);
}
- @Test
- public void testRefreshCatalog() throws Exception {
- CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
- Assertions.assertFalse(table.isObjectCreated());
- table.makeSureInitialized();
- Assertions.assertTrue(table.isObjectCreated());
- RefreshCatalogStmt refreshCatalogStmt = new
RefreshCatalogStmt("test1", null);
- Assertions.assertTrue(refreshCatalogStmt.isInvalidCache());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertTrue(table.isObjectCreated());
- test1.getDbNullable("db1").getTables();
- Assertions.assertFalse(table.isObjectCreated());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalCatalog) test1).isInitialized());
- table.makeSureInitialized();
- Assertions.assertTrue(((ExternalCatalog) test1).isInitialized());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalCatalog) test1).isInitialized());
- }
-
- @Test
- public void testRefreshDatabase() throws Exception {
- CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
- TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
- Assertions.assertFalse(table.isObjectCreated());
- table.makeSureInitialized();
- Assertions.assertTrue(table.isObjectCreated());
- RefreshDbStmt refreshDbStmt = new RefreshDbStmt("test1", "db1", null);
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertTrue(table.isObjectCreated());
- test1.getDbNullable("db1").getTables();
- Assertions.assertFalse(table.isObjectCreated());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalDatabase)
test1.getDbNullable("db1")).isInitialized());
- table.makeSureInitialized();
- Assertions.assertTrue(((ExternalDatabase)
test1.getDbNullable("db1")).isInitialized());
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt);
- } catch (Exception e) {
- // Do nothing
- }
- }
-
@Test
public void testRefreshTable() throws Exception {
CatalogIf test1 = env.getCatalogMgr().getCatalog("test1");
TestExternalTable table = (TestExternalTable)
test1.getDbNullable("db1").getTable("tbl11").get();
Assertions.assertFalse(table.isObjectCreated());
+ long l1 = table.getLastUpdateTime();
+ Assertions.assertTrue(l1 == 0);
table.makeSureInitialized();
Assertions.assertTrue(table.isObjectCreated());
+ long l2 = table.getLastUpdateTime();
+ Assertions.assertTrue(l2 == l1);
RefreshTableStmt refreshTableStmt = new RefreshTableStmt(new
TableName("test1", "db1", "tbl11"));
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshTableStmt);
@@ -146,6 +82,12 @@ public class RefreshTableTest extends TestWithFeService {
// Do nothing
}
Assertions.assertFalse(table.isObjectCreated());
+ long l3 = table.getLastUpdateTime();
+ Assertions.assertTrue(l3 == l2);
+ table.getFullSchema();
+ // only table.getFullSchema() can change table.lastUpdateTime
+ long l4 = table.getLastUpdateTime();
+ Assertions.assertTrue(l4 > l3);
}
public static class RefreshTableProvider implements
TestExternalCatalog.TestCatalogProvider {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
index 07e8a679dc..b3052643d2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
@@ -193,11 +193,13 @@ public class DbsProcDirTest {
Assert.assertTrue(result instanceof BaseProcResult);
Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum",
"Size", "Quota",
- "LastConsistencyCheckTime", "ReplicaCount",
"ReplicaQuota", "TransactionQuota"),
+ "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota",
"TransactionQuota", "LastUpdateTime"),
result.getColumnNames());
List<List<String>> rows = Lists.newArrayList();
- rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(),
"0", "0.000 ", "1024.000 TB", FeConstants.null_string, "0", "1073741824",
"1000"));
- rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(),
"0", "0.000 ", "1024.000 TB", FeConstants.null_string, "0", "1073741824",
"1000"));
+ rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(),
"0", "0.000 ", "1024.000 TB",
+ FeConstants.null_string, "0", "1073741824", "1000",
FeConstants.null_string));
+ rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(),
"0", "0.000 ", "1024.000 TB",
+ FeConstants.null_string, "0", "1073741824", "1000",
FeConstants.null_string));
Assert.assertEquals(rows, result.getRows());
}
@@ -228,8 +230,8 @@ public class DbsProcDirTest {
dir = new DbsProcDir(env, catalog);
result = dir.fetchResult();
Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum",
"Size", "Quota",
- "LastConsistencyCheckTime", "ReplicaCount",
"ReplicaQuota", "TransactionQuota"),
- result.getColumnNames());
+ "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota",
"TransactionQuota", "LastUpdateTime"),
+ result.getColumnNames());
List<List<String>> rows = Lists.newArrayList();
Assert.assertEquals(rows, result.getRows());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
