This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ec264b8f78d [fix](txn insert) Forbid delete condition in txn insert
(#37135)
ec264b8f78d is described below
commit ec264b8f78df742067ed178b927ac5b258ff0f33
Author: meiyi <myime...@gmail.com>
AuthorDate: Tue Jul 2 20:48:20 2024 +0800
[fix](txn insert) Forbid delete condition in txn insert (#37135)
## Proposed changes
Currently, there are two kind of delete in doris, one is delete where
which is implemented via fast path, while the other one is delete using
which is implemented via insert into alike.
The first one can read writings in the same txn while the second one can
not. This is confusing for users, so we just disable delete after insert
and update now.
---
.../java/org/apache/doris/load/TxnDeleteJob.java | 2 +-
.../commands/insert/OlapTxnInsertExecutor.java | 2 +-
.../apache/doris/transaction/TransactionEntry.java | 19 ++++------
regression-test/data/insert_p0/txn_insert.out | 22 ++++++++++++
.../insert_p0/txn_insert_with_schema_change.out | 42 ----------------------
regression-test/suites/insert_p0/txn_insert.groovy | 36 ++++++++++---------
.../insert_p0/txn_insert_with_schema_change.groovy | 4 +--
.../txn_insert_concurrent_insert_ud.groovy | 6 ++--
.../txn_insert_concurrent_insert_update.groovy | 6 ++--
.../insert_p2/txn_insert_with_schema_change.groovy | 4 +--
10 files changed, 61 insertions(+), 82 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/TxnDeleteJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/TxnDeleteJob.java
index 5a508b910c5..916a8395487 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/TxnDeleteJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/TxnDeleteJob.java
@@ -44,7 +44,7 @@ public class TxnDeleteJob extends DeleteJob {
@Override
public long beginTxn() throws Exception {
TransactionEntry txnEntry = ConnectContext.get().getTxnEntry();
- this.transactionId = txnEntry.beginTransaction(targetTbl);
+ this.transactionId = txnEntry.beginTransaction(targetTbl,
SubTransactionType.DELETE);
this.label = txnEntry.getLabel();
return this.transactionId;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapTxnInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapTxnInsertExecutor.java
index b16930a1a2a..ebe0a318e19 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapTxnInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapTxnInsertExecutor.java
@@ -58,7 +58,7 @@ public class OlapTxnInsertExecutor extends OlapInsertExecutor
{
throw new AnalysisException("Transaction insert expect label "
+ txnEntry.getLabel()
+ ", but got " + this.labelName);
}
- this.txnId = txnEntry.beginTransaction(table);
+ this.txnId = txnEntry.beginTransaction(table,
SubTransactionType.INSERT);
this.labelName = txnEntry.getLabel();
} catch (Exception e) {
throw new AnalysisException("begin transaction failed. " +
e.getMessage(), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
index e9db8e3f58f..6771d9c3156 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
@@ -182,7 +182,7 @@ public class TransactionEntry {
}
// Used for insert into select, return the sub_txn_id for this insert
- public long beginTransaction(TableIf table) throws Exception {
+ public long beginTransaction(TableIf table, SubTransactionType
subTransactionType) throws Exception {
if (isInsertValuesTxnBegan()) {
// FIXME: support mix usage of `insert into values` and `insert
into select`
throw new AnalysisException(
@@ -226,6 +226,12 @@ public class TransactionEntry {
throw new AnalysisException(
"Transaction insert must be in the same database,
expect db_id=" + this.database.getId());
}
+ // for delete type, make sure there is no insert for the same table
+ if (subTransactionType == SubTransactionType.DELETE &&
subTransactionStates.stream()
+ .anyMatch(s -> s.getTable().getId() == table.getId()
+ && s.getSubTransactionType() ==
SubTransactionType.INSERT)) {
+ throw new AnalysisException("Can not delete because there is a
insert operation for the same table");
+ }
long subTxnId;
if (Config.isCloudMode()) {
TUniqueId queryId = ConnectContext.get().queryId();
@@ -348,17 +354,6 @@ public class TransactionEntry {
.collect(Collectors.toList());
transactionState.setTableIdList(tableIds);
}
- subTransactionStates.sort((s1, s2) -> {
- if (s1.getSubTransactionType() == SubTransactionType.INSERT
- && s2.getSubTransactionType() ==
SubTransactionType.DELETE) {
- return 1;
- } else if (s1.getSubTransactionType() ==
SubTransactionType.DELETE
- && s2.getSubTransactionType() ==
SubTransactionType.INSERT) {
- return -1;
- } else {
- return Long.compare(s1.getSubTransactionId(),
s2.getSubTransactionId());
- }
- });
LOG.info("subTransactionStates={}", subTransactionStates);
transactionState.setSubTxnIds(subTransactionStates.stream().map(SubTransactionState::getSubTransactionId)
.collect(Collectors.toList()));
diff --git a/regression-test/data/insert_p0/txn_insert.out
b/regression-test/data/insert_p0/txn_insert.out
index c1278f7fe2e..9232eba9da3 100644
--- a/regression-test/data/insert_p0/txn_insert.out
+++ b/regression-test/data/insert_p0/txn_insert.out
@@ -1112,6 +1112,14 @@
1 2.2 abc [] []
1 2.2 abc [] []
1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
100 2.2 abc [] []
100 2.2 abc [] []
100 2.2 abc [] []
@@ -1129,6 +1137,12 @@
2 3.3 xyz [1] [1, 0]
2 3.3 xyz [1] [1, 0]
2 3.3 xyz [1] [1, 0]
+2 3.3 xyz [1] [1, 0]
+2 3.3 xyz [1] [1, 0]
+2 3.3 xyz [1] [1, 0]
+2 3.3 xyz [1] [1, 0]
+2 3.3 xyz [1] [1, 0]
+2 3.3 xyz [1] [1, 0]
-- !select48 --
\N \N \N [null] [null, 0]
@@ -1141,6 +1155,14 @@
1 2.2 abc [] []
1 2.2 abc [] []
1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
+1 2.2 abc [] []
100 2.2 abc [] []
101 2.2 abc [] []
2 3.3 xyz [1] [1, 0]
diff --git a/regression-test/data/insert_p0/txn_insert_with_schema_change.out
b/regression-test/data/insert_p0/txn_insert_with_schema_change.out
index ded5f777182..21c5c2025ea 100644
--- a/regression-test/data/insert_p0/txn_insert_with_schema_change.out
+++ b/regression-test/data/insert_p0/txn_insert_with_schema_change.out
@@ -119,45 +119,3 @@
5 25 15
5 25 15
--- !select1 --
-3 23 13
-4 24 14
-5 25 15
-
--- !select2 --
-3 23 13
-4 24 14
-5 25 15
-
--- !select3 --
-3 23 13
-4 24 14
-4 24 14
-5 25 15
-5 25 15
-
--- !select4 --
-3 23 13
-4 24 14
-4 24 14
-5 25 15
-5 25 15
-
--- !select5 --
-3 23 13
-4 24 14
-4 24 14
-4 24 14
-5 25 15
-5 25 15
-5 25 15
-
--- !select6 --
-3 23 13
-4 24 14
-4 24 14
-4 24 14
-5 25 15
-5 25 15
-5 25 15
-
diff --git a/regression-test/suites/insert_p0/txn_insert.groovy
b/regression-test/suites/insert_p0/txn_insert.groovy
index c403f4c132c..088e1134900 100644
--- a/regression-test/suites/insert_p0/txn_insert.groovy
+++ b/regression-test/suites/insert_p0/txn_insert.groovy
@@ -381,10 +381,19 @@ suite("txn_insert") {
order_qt_select46 """select * from ${table}_1"""
sql """ begin; """
sql """ insert into ${table}_0 select * from ${table}_1 where k1 =
1 or k1 = 2; """
- sql """ delete from ${table}_0 where k1 = 1 or k1 = 2; """
+ test {
+ sql """ delete from ${table}_0 where k1 = 1 or k1 = 2; """
+ exception "Can not delete because there is a insert operation
for the same table"
+ }
sql """ insert into ${table}_1 select * from ${table}_0 where k1 =
1 or k1 = 2; """
- sql """ delete from ${table}_0 where k1 = 1 or k1 = 2; """
- sql """ delete from ${table}_1 where k1 = 1; """
+ test {
+ sql """ delete from ${table}_0 where k1 = 1 or k1 = 2; """
+ exception "Can not delete because there is a insert operation
for the same table"
+ }
+ test {
+ sql """ delete from ${table}_1 where k1 = 1; """
+ exception "Can not delete because there is a insert operation
for the same table"
+ }
sql """ commit; """
sql "sync"
order_qt_select47 """select * from ${table}_0"""
@@ -684,18 +693,10 @@ suite("txn_insert") {
using txn_insert_dt2 join txn_insert_dt3 on txn_insert_dt2.id
= txn_insert_dt3.id
where txn_insert_dt4.id = txn_insert_dt2.id;
"""
- sql """
- delete from txn_insert_dt2 where id = 1;
- """
- sql """
- delete from txn_insert_dt2 where id = 5;
- """
- sql """
- delete from txn_insert_dt5 partition(p_20000102) where id = 1;
- """
- sql """
- delete from txn_insert_dt5 partition(p_20000102) where id = 5;
- """
+ sql """ delete from txn_insert_dt2 where id = 1; """
+ sql """ delete from txn_insert_dt2 where id = 5; """
+ sql """ delete from txn_insert_dt5 partition(p_20000102) where id
= 1; """
+ sql """ delete from txn_insert_dt5 partition(p_20000102) where id
= 5; """
sql """ commit """
sql """ insert into txn_insert_dt2 VALUES (6, '2000-01-10', 10,
'10', 10.0) """
sql """ insert into txn_insert_dt5 VALUES (6, '2000-01-10', 10,
'10', 10.0) """
@@ -736,7 +737,10 @@ suite("txn_insert") {
sql """ insert into ${unique_table}_2(id, score) select id,
score from ${unique_table}_0; """
sql """ insert into ${unique_table}_2(id, score) select id,
score from ${unique_table}_1; """
sql """ update ${unique_table}_2 set score = score + 100 where
id in (select id from ${unique_table}_0); """
- sql """ delete from ${unique_table}_2 where id <= 1; """
+ test {
+ sql """ delete from ${unique_table}_2 where id <= 1; """
+ // exception "Can not delete because there is a insert
operation for the same table"
+ }
sql """ commit """
sql """ insert into ${unique_table}_3(id, score) select id,
score from ${unique_table}_0; """
diff --git
a/regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy
b/regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy
index ec9bc55a729..a247183314d 100644
--- a/regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy
+++ b/regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy
@@ -112,8 +112,8 @@ suite("txn_insert_with_schema_change") {
"insert into ${table}_0(id, name, score) select * from
${table}_4;"],
["delete from ${table}_1 where id = 0 or id = 3;",
"insert into ${table}_1(id, name, score) select * from
${table}_4;"],
- ["insert into ${table}_2(id, name, score) select * from
${table}_4;",
- "delete from ${table}_2 where id = 0 or id = 3;"]
+ /*["insert into ${table}_2(id, name, score) select * from
${table}_4;",
+ "delete from ${table}_2 where id = 0 or id = 3;"]*/
]
for (int i = 0; i < sqls.size(); i++) {
diff --git
a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy
b/regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy
index 787c556dc3d..71599521a3f 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy
+++ b/regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy
@@ -120,8 +120,8 @@ suite("txn_insert_concurrent_insert_ud") {
"insert into ${tableName}_0 select * from ${tableName}_1 where
L_ORDERKEY >= 1000000 and L_ORDERKEY < 2000000;",
"update ${tableName}_0 set L_QUANTITY = L_QUANTITY + 10 where
L_ORDERKEY < 1000000;",
"update ${tableName}_0 set ${tableName}_0.L_QUANTITY = 100 where
${tableName}_0.L_ORDERKEY in (select L_ORDERKEY from ${tableName}_1 where
L_ORDERKEY >= 2000000 and L_ORDERKEY < 3000000);",
- "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_1 where L_ORDERKEY >= 2000000);",
- "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_2 where L_ORDERKEY >= 3000000 and
L_ORDERKEY < 4000000);",
+ // "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_1 where L_ORDERKEY >= 2000000);",
+ // "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_2 where L_ORDERKEY >= 3000000 and
L_ORDERKEY < 4000000);",
]
def txn_insert = { ->
try (Connection conn = DriverManager.getConnection(url,
context.config.jdbcUser, context.config.jdbcPassword);
@@ -163,7 +163,7 @@ suite("txn_insert_concurrent_insert_ud") {
logger.info("error num: " + errors.size() + ", errors: " + errors)
- def t0_row_count = 2000495 // 5000226 or 6001215
+ def t0_row_count = 6001215 // 2000495 or 5000226
def result = sql """ select count() from ${tableName}_0 """
logger.info("${tableName}_0 row count: ${result}, expected >=
${t0_row_count}")
diff --git
a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy
b/regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy
index 2eb5c4b30ef..a31213969cc 100644
---
a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy
+++
b/regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy
@@ -128,8 +128,8 @@ suite("txn_insert_concurrent_insert_update") {
"select L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER,
L_QUANTITY from ${tableName}_1 where L_ORDERKEY >= 1000000 and L_ORDERKEY <
2000000;",
"update ${tableName}_0 set L_QUANTITY = L_QUANTITY + 10 where
L_ORDERKEY < 1000000;",
"update ${tableName}_0 set ${tableName}_0.L_QUANTITY = 100 where
${tableName}_0.L_ORDERKEY in (select L_ORDERKEY from ${tableName}_1 where
L_ORDERKEY >= 2000000 and L_ORDERKEY < 3000000);",
- "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_1 where L_ORDERKEY >= 2000000);",
- "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_2 where L_ORDERKEY >= 3000000 and
L_ORDERKEY < 4000000);",
+ // "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_1 where L_ORDERKEY >= 2000000);",
+ // "delete from ${tableName}_0 where ${tableName}_0.L_ORDERKEY in
(select L_ORDERKEY from ${tableName}_2 where L_ORDERKEY >= 3000000 and
L_ORDERKEY < 4000000);",
]
def txn_insert = { ->
try (Connection conn = DriverManager.getConnection(url,
context.config.jdbcUser, context.config.jdbcPassword);
@@ -175,7 +175,7 @@ suite("txn_insert_concurrent_insert_update") {
logger.info("error num: " + errors.size() + ", errors: " + errors)
- def t0_row_count = 2000495 // 5000226 or 6001215
+ def t0_row_count = 6001215 // 2000495 or 5000226
def result = sql """ select count() from ${tableName}_0 """
logger.info("${tableName}_0 row count: ${result}, expected >=
${t0_row_count}")
diff --git
a/regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy
b/regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy
index 1f45955e81d..de11936bb53 100644
--- a/regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy
+++ b/regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy
@@ -157,8 +157,8 @@ suite("txn_insert_with_schema_change") {
"insert into ${tableName}_0(L_ORDERKEY, L_PARTKEY, L_SUPPKEY,
L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, L_RETURNFLAG,
L_LINESTATUS, L_SHIPDATE, L_COMMITDATE, L_RECEIPTDATE, L_SHIPINSTRUCT,
L_SHIPMODE, L_COMMENT) select * from ${tableName}_3;"],
["delete from ${tableName}_1 where L_ORDERKEY < 50000;",
"insert into ${tableName}_1(L_ORDERKEY, L_PARTKEY, L_SUPPKEY,
L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, L_RETURNFLAG,
L_LINESTATUS, L_SHIPDATE, L_COMMITDATE, L_RECEIPTDATE, L_SHIPINSTRUCT,
L_SHIPMODE, L_COMMENT) select * from ${tableName}_3;"],
- ["insert into ${tableName}_2(L_ORDERKEY, L_PARTKEY, L_SUPPKEY,
L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, L_RETURNFLAG,
L_LINESTATUS, L_SHIPDATE, L_COMMITDATE, L_RECEIPTDATE, L_SHIPINSTRUCT,
L_SHIPMODE, L_COMMENT) select * from ${tableName}_3;",
- "delete from ${tableName}_2 where L_ORDERKEY < 50000;"]
+ /*["insert into ${tableName}_2(L_ORDERKEY, L_PARTKEY, L_SUPPKEY,
L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, L_RETURNFLAG,
L_LINESTATUS, L_SHIPDATE, L_COMMITDATE, L_RECEIPTDATE, L_SHIPINSTRUCT,
L_SHIPMODE, L_COMMENT) select * from ${tableName}_3;",
+ "delete from ${tableName}_2 where L_ORDERKEY < 50000;"]*/
]
def expected_row_count = [
[6001215 * 5, 6001215 * 8, 6001215 * 11, 6001215 * 14, 6001215 *
17],
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
