This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 67f1ae8a12f [Enhancement] support to erase all tables and partitions
with same DbId while drop a database which is not in catalog recycle bin
(#35750)
67f1ae8a12f is described below
commit 67f1ae8a12f18615ecee185ccaf1694f543d0e0b
Author: Liu Zhenlong <49094455+dragonliu2...@users.noreply.github.com>
AuthorDate: Sun Jun 2 10:09:55 2024 +0800
[Enhancement] support to erase all tables and partitions with same DbId
while drop a database which is not in catalog recycle bin (#35750)
## Proposed changes
Issue Number: close #35748
<!--Describe your changes.-->
When drop a database which is not in catalog recycle bin, remove all
tables and partitions in catalog recycle bin with same DbId, not simply
reporting errors. Such as:
```sql
mysql> show catalog recycle bin;
+-----------+------+-------+---------+-------------+---------------------+----------+----------------+
| Type | Name | DbId | TableId | PartitionId | DropTime |
DataSize | RemoteDataSize |
+-----------+------+-------+---------+-------------+---------------------+----------+----------------+
| Partition | p30 | 12056 | 12258 | 12257 | 2024-05-31 22:36:45 |
0.000 | 0.000 |
+-----------+------+-------+---------+-------------+---------------------+----------+----------------+
1 row in set (0.01 sec)
mysql> drop catalog recycle bin where 'dbid' = 12056;
Query OK, 0 rows affected (0.01 sec)
mysql> show catalog recycle bin;
Empty set (0.00 sec)
```
When drop a table which is not in catalog recycle bin, the treatment is
similar.
---
.../apache/doris/catalog/CatalogRecycleBin.java | 75 ++++++++++++----------
.../test_drop_catalog_recycle_bin.groovy | 31 ++++++++-
2 files changed, 69 insertions(+), 37 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index a344176df6b..468a66ed400 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -909,24 +909,22 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
// erase database in catalog recycle bin instantly
public synchronized void eraseDatabaseInstantly(long dbId) throws
DdlException {
- // 1. find dbInfo to erase
+ // 1. find dbInfo and erase db
RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId);
- if (dbInfo == null) {
- throw new DdlException("Unknown database id '" + dbId + "'");
- }
-
- // 2. erase db
- Env.getCurrentEnv().eraseDatabase(dbId, true);
+ if (dbInfo != null) {
+ // erase db
+ Env.getCurrentEnv().eraseDatabase(dbId, true);
- // 3. erase db from idToDatabase and idToRecycleTime
- idToDatabase.remove(dbId);
- idToRecycleTime.remove(dbId);
+ // erase db from idToDatabase and idToRecycleTime
+ idToDatabase.remove(dbId);
+ idToRecycleTime.remove(dbId);
- // 4. log for erase db
- String dbName = dbInfo.getDb().getName();
- LOG.info("erase db[{}]: {}", dbId, dbName);
+ // log for erase db
+ String dbName = dbInfo.getDb().getName();
+ LOG.info("erase db[{}]: {}", dbId, dbName);
+ }
- // 5. remove all tables with the same dbId
+ // 2. remove all tables with the same dbId
List<Long> tableIdToErase = Lists.newArrayList();
Iterator<Map.Entry<Long, RecycleTableInfo>> tableIterator =
idToTable.entrySet().iterator();
while (tableIterator.hasNext()) {
@@ -940,7 +938,7 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
eraseTableInstantly(tableId);
}
- // 6. remove all partitions with the same dbId
+ // 3. remove all partitions with the same dbId
List<Long> partitionIdToErase = Lists.newArrayList();
Iterator<Map.Entry<Long, RecyclePartitionInfo>> partitionIterator =
idToPartition.entrySet().iterator();
while (partitionIterator.hasNext()) {
@@ -953,34 +951,36 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
for (Long partitionId : partitionIdToErase) {
erasePartitionInstantly(partitionId);
}
+
+ // 4. determine if nothing is deleted
+ if (dbInfo == null && tableIdToErase.isEmpty() &&
partitionIdToErase.isEmpty()) {
+ throw new DdlException("Unknown database id '" + dbId + "'");
+ }
}
// erase table in catalog recycle bin instantly
public synchronized void eraseTableInstantly(long tableId) throws
DdlException {
- // 1. find tableInfo to erase
+ // 1. find tableInfo and erase table
RecycleTableInfo tableInfo = idToTable.get(tableId);
- if (tableInfo == null) {
- throw new DdlException("Unknown table id '" + tableId + "'");
- }
-
- // 2. erase table
- long dbId = tableInfo.getDbId();
- Table table = tableInfo.getTable();
- if (table.getType() == TableType.OLAP || table.getType() ==
TableType.MATERIALIZED_VIEW) {
- Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, false);
- }
-
- // 3. erase table from idToTable and idToRecycleTime
- idToTable.remove(tableId);
- idToRecycleTime.remove(tableId);
+ if (tableInfo != null) {
+ // erase table
+ long dbId = tableInfo.getDbId();
+ Table table = tableInfo.getTable();
+ if (table.getType() == TableType.OLAP || table.getType() ==
TableType.MATERIALIZED_VIEW) {
+ Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, false);
+ }
- // 4. log for erase table
- String tableName = table.getName();
- Env.getCurrentEnv().getEditLog().logEraseTable(tableId);
- LOG.info("erase db[{}]'s table[{}]: {}", dbId, tableId, tableName);
+ // erase table from idToTable and idToRecycleTime
+ idToTable.remove(tableId);
+ idToRecycleTime.remove(tableId);
+ // log for erase table
+ String tableName = table.getName();
+ Env.getCurrentEnv().getEditLog().logEraseTable(tableId);
+ LOG.info("erase db[{}]'s table[{}]: {}", dbId, tableId, tableName);
+ }
- // 5. erase all partitions with the same tableId
+ // 2. erase all partitions with the same tableId
List<Long> partitionIdToErase = Lists.newArrayList();
Iterator<Map.Entry<Long, RecyclePartitionInfo>> partitionIterator =
idToPartition.entrySet().iterator();
while (partitionIterator.hasNext()) {
@@ -993,6 +993,11 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
for (Long partitionId : partitionIdToErase) {
erasePartitionInstantly(partitionId);
}
+
+ // 3. determine if nothing is deleted
+ if (tableInfo == null && partitionIdToErase.isEmpty()) {
+ throw new DdlException("Unknown table id '" + tableId + "'");
+ }
}
// erase partition in catalog recycle bin instantly
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
b/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
index f3b5a987ee4..40b1175c856 100644
---
a/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
+++
b/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
@@ -65,7 +65,6 @@ suite("test_drop_catalog_recycle_bin") {
sql "use `test_drop_catalog_recycle_bin_db`"
sql "ALTER TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP
PARTITION p1000;"
- sql "ALTER TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP
PARTITION p111;"
pre_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p1000" """
assertTrue(pre_res.size() > 0)
@@ -74,12 +73,26 @@ suite("test_drop_catalog_recycle_bin") {
cur_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p1000" """
assertTrue(pre_res.size() - cur_res.size() == 1)
+ // test drop table not in catalog recycle bin
+ sql "use `test_drop_catalog_recycle_bin_db`"
+ sql "ALTER TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP
PARTITION p111;"
+
+ pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
+ assertTrue(pre_pt_res.size() > 0)
+ table_id = pre_res[0][3]
+ sql "DROP CATALOG RECYCLE BIN WHERE 'TableId' = ${table_id};"
+ cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
+ assertTrue(pre_pt_res.size() - cur_pt_res.size() == 1)
+
// test drop table in catalog recycle bin
+ sql "use `test_drop_catalog_recycle_bin_db`"
+ sql "ALTER TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP
PARTITION p222;"
sql "DROP TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1`;"
pre_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME =
"test_drop_catalog_recycle_bin_tb1" """
assertTrue(pre_tb_res.size() > 0)
- pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
+ pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p222" """
+ assertTrue(pre_pt_res.size() > 0)
table_id = pre_res[0][3]
sql "DROP CATALOG RECYCLE BIN WHERE 'TableId' = ${table_id};"
cur_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME =
"test_drop_catalog_recycle_bin_tb1" """
@@ -87,6 +100,20 @@ suite("test_drop_catalog_recycle_bin") {
cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
assertTrue(pre_pt_res.size() - cur_pt_res.size() == 1)
+ // test drop db not in catalog recycle bin
+ sql "ALTER TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb2` DROP
PARTITION p111;"
+
+ pre_db_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME =
"test_drop_catalog_recycle_bin_db" """
+ assertTrue(pre_db_res.size() == 0)
+ pre_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME =
"test_drop_catalog_recycle_bin_tb2" """
+ assertTrue(pre_tb_res.size() == 0)
+ pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
+ assertTrue(pre_pt_res.size() > 0)
+ db_id = pre_res[0][2]
+ sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${db_id};"
+ cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p222" """
+ assertTrue(pre_pt_res.size() - cur_pt_res.size() == 1)
+
// test drop db in catalog recycle bin
sql "ALTER TABLE
`test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb2` DROP
PARTITION p1000;"
sql """ DROP DATABASE `test_drop_catalog_recycle_bin_db` """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
