This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9b500faa0c1 [fix](create table) create table fail not write drop table
editlog #37488 (#37506)
9b500faa0c1 is described below
commit 9b500faa0c1b211256c1a39b3620f41231075a77
Author: yujun <yu.jun.re...@gmail.com>
AuthorDate: Tue Jul 9 13:44:43 2024 +0800
[fix](create table) create table fail not write drop table editlog #37488
(#37506)
cherry pick from #37488
---
.../apache/doris/datasource/InternalCatalog.java | 29 ++++++++++-----
.../test_create_table_exception.groovy | 43 +++++++++++++++-------
2 files changed, 49 insertions(+), 23 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 06b9c33eea9..19672e0ccc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -973,13 +973,13 @@ public class InternalCatalog implements
CatalogIf<Database> {
return true;
}
- public void dropTable(Database db, long tableId, boolean isForceDrop,
+ private void dropTable(Database db, long tableId, boolean isForceDrop,
boolean isReplay,
Long recycleTime) throws MetaNotFoundException {
Table table = db.getTableOrMetaException(tableId);
db.writeLock();
table.writeLock();
try {
- unprotectDropTable(db, table, isForceDrop, true, recycleTime);
+ unprotectDropTable(db, table, isForceDrop, isReplay, recycleTime);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(),
db.getId(), tableId);
Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
} finally {
@@ -990,7 +990,7 @@ public class InternalCatalog implements CatalogIf<Database>
{
public void replayDropTable(Database db, long tableId, boolean isForceDrop,
Long recycleTime) throws MetaNotFoundException {
- dropTable(db, tableId, isForceDrop, recycleTime);
+ dropTable(db, tableId, isForceDrop, true, recycleTime);
}
public void replayEraseTable(long tableId) {
@@ -2639,6 +2639,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
// if failed in any step, use this set to do clear things
Set<Long> tabletIdSet = new HashSet<>();
// create partition
+ boolean hadLogEditCreateTable = false;
try {
if (partitionInfo.getType() == PartitionType.UNPARTITIONED) {
if (storagePolicy.equals("") && properties != null &&
!properties.isEmpty()) {
@@ -2762,11 +2763,6 @@ public class InternalCatalog implements
CatalogIf<Database> {
if (!result.first) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
}
- if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) {
- LOG.info("debug point FE.createOlapTable.exception, throw e");
- // not commit, not log edit
- throw new DdlException("debug point
FE.createOlapTable.exception");
- }
if (result.second) {
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
@@ -2779,6 +2775,9 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
LOG.info("duplicate create table[{};{}], skip next steps",
tableName, tableId);
} else {
+ // if table not exists, then db.createTableWithLock will write
an editlog.
+ hadLogEditCreateTable = true;
+
// we have added these index to memory, only need to persist
here
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
GroupId groupId =
Env.getCurrentColocateIndex().getGroup(tableId);
@@ -2797,17 +2796,27 @@ public class InternalCatalog implements
CatalogIf<Database> {
.createOrUpdateRuntimeInfo(tableId,
DynamicPartitionScheduler.LAST_UPDATE_TIME,
TimeUtils.getCurrentFormatTime());
}
+
+ if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) {
+ LOG.info("debug point FE.createOlapTable.exception, throw e");
+ throw new DdlException("debug point
FE.createOlapTable.exception");
+ }
} catch (DdlException e) {
LOG.warn("create table failed {} - {}", tabletIdSet,
e.getMessage());
for (Long tabletId : tabletIdSet) {
Env.getCurrentInvertedIndex().deleteTablet(tabletId);
}
- // only remove from memory, because we have not persist it
+ // edit log write DropTableInfo will result in deleting colocate
group,
+ // but follow fe may need wait 30s (recycle bin mgr run every 30s).
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
Env.getCurrentColocateIndex().removeTable(tableId);
}
try {
- dropTable(db, tableId, true, 0L);
+ dropTable(db, tableId, true, false, 0L);
+ if (hadLogEditCreateTable) {
+ DropInfo info = new DropInfo(db.getId(), tableId,
olapTable.getName(), -1L, true, 0L);
+ Env.getCurrentEnv().getEditLog().logDropTable(info);
+ }
} catch (Exception ex) {
LOG.warn("drop table", ex);
}
diff --git
a/regression-test/suites/partition_p0/test_create_table_exception.groovy
b/regression-test/suites/partition_p0/test_create_table_exception.groovy
index 49cadcd3af4..96f097c7670 100644
--- a/regression-test/suites/partition_p0/test_create_table_exception.groovy
+++ b/regression-test/suites/partition_p0/test_create_table_exception.groovy
@@ -31,9 +31,9 @@ suite("test_create_table_exception") {
def table3 = "dynamic_partition_table"
try {
GetDebugPoint().enableDebugPointForAllFEs('FE.createOlapTable.exception', null)
- def createTable = { ->
+ def createTable = { tableIdx ->
try_sql """
- CREATE TABLE $table1 (
+ CREATE TABLE ${table1}_${tableIdx} (
`k1` int(11) NULL,
`k2` int(11) NULL
)
@@ -41,12 +41,13 @@ suite("test_create_table_exception") {
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 10
PROPERTIES (
+ "colocate_with" = "col_grp_${tableIdx}",
"replication_num"="3"
);
"""
try_sql """
- CREATE TABLE IF NOT EXISTS $table2 (
+ CREATE TABLE IF NOT EXISTS ${table2}_${tableIdx} (
lo_orderdate int(11) NOT NULL COMMENT "",
lo_orderkey bigint(20) NOT NULL COMMENT "",
lo_linenumber bigint(20) NOT NULL COMMENT "",
@@ -79,7 +80,7 @@ suite("test_create_table_exception") {
"""
try_sql """
- CREATE TABLE $table3 (
+ CREATE TABLE ${table3}_${tableIdx} (
time date,
key1 int,
key2 int,
@@ -109,19 +110,35 @@ suite("test_create_table_exception") {
);
"""
}
- createTable()
+ createTable(1)
def result = sql """show tables;"""
assertEquals(result.size(), 0)
+
+ def checkResult = { ->
+ def tables = sql """show tables;"""
+ log.info("tables=" + tables)
+ assertEquals(3, tables.size())
+
+ def groups = sql """ show proc "/colocation_group" """
+ log.info("groups=" + groups)
+ assertEquals(1, groups.size())
+ }
+
GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception')
- createTable()
- result = sql """show tables;"""
- log.info(result.toString())
- assertEquals(result.size(), 3)
+ createTable(2)
+ checkResult()
+
+ sleep 1000
+ cluster.restartFrontends(cluster.getMasterFe().index)
+ sleep 32_000
+ def newMasterFe = cluster.getMasterFe()
+ def newMasterFeUrl =
"jdbc:mysql://${newMasterFe.host}:${newMasterFe.queryPort}/?useLocalSessionState=false&allowLoadLocalInfile=false"
+ newMasterFeUrl = context.config.buildUrlWithDb(newMasterFeUrl,
context.dbName)
+ connect('root', '', newMasterFeUrl) {
+ checkResult()
+ }
+
} finally {
-
GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception')
- sql """drop table if exists ${table1}"""
- sql """drop table if exists ${table2}"""
- sql """drop table if exists ${table3}"""
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
