This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 64880a10d67 [branch-2.1] Picks "[Fix](partial update) Fix partial
update failed when merge_type=MERGE #40730" (#40951)
64880a10d67 is described below
commit 64880a10d6749584e96ddb60448075f0d0e18311
Author: bobhan1 <[email protected]>
AuthorDate: Fri Sep 20 00:02:17 2024 +0800
[branch-2.1] Picks "[Fix](partial update) Fix partial update failed when
merge_type=MERGE #40730" (#40951)
picks https://github.com/apache/doris/pull/40730
---
.../apache/doris/planner/StreamLoadPlanner.java | 6 +-
.../unique_with_mow_p0/partial_update/merge1.csv | 6 +
.../unique_with_mow_p0/partial_update/merge2.csv | 2 +
.../unique_with_mow_p0/partial_update/merge3.csv | 6 +
.../unique_with_mow_p0/partial_update/merge4.csv | 2 +
.../unique_with_mow_p0/partial_update/merge5.csv | 2 +
.../unique_with_mow_p0/partial_update/merge6.csv | 2 +
.../unique_with_mow_p0/partial_update/merge7.csv | 2 +
.../unique_with_mow_p0/partial_update/merge8.csv | 2 +
.../test_partial_update_merge_type.out | 341 +++++++++++++++++++++
.../test_partial_update_merge_type.groovy | 185 +++++++++++
11 files changed, 554 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
index d89f7b55794..97da7743c2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
@@ -189,7 +189,8 @@ public class StreamLoadPlanner {
}
}
}
- if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE) {
+ if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE
+ || taskInfo.getMergeType() == LoadTask.MergeType.MERGE) {
partialUpdateInputColumns.add(Column.DELETE_SIGN);
}
}
@@ -440,7 +441,8 @@ public class StreamLoadPlanner {
}
}
}
- if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE) {
+ if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE
+ || taskInfo.getMergeType() == LoadTask.MergeType.MERGE) {
partialUpdateInputColumns.add(Column.DELETE_SIGN);
}
}
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge1.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge1.csv
new file mode 100644
index 00000000000..24364822bcc
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge1.csv
@@ -0,0 +1,6 @@
+1,10,0
+2,20,1
+3,30,0
+4,40,1
+10,999,0
+11,888,1
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge2.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge2.csv
new file mode 100644
index 00000000000..8292fab3180
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge2.csv
@@ -0,0 +1,2 @@
+1,11
+5,50
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge3.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge3.csv
new file mode 100644
index 00000000000..2aad468b297
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge3.csv
@@ -0,0 +1,6 @@
+1,10,2,0
+2,20,2,1
+3,30,2,0
+4,40,2,1
+10,999,2,0
+11,888,2,1
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge4.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge4.csv
new file mode 100644
index 00000000000..cec571fcb9d
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge4.csv
@@ -0,0 +1,2 @@
+1,11,4
+5,50,4
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge5.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge5.csv
new file mode 100644
index 00000000000..4170f9d8b26
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge5.csv
@@ -0,0 +1,2 @@
+6,1
+11,0
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge6.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge6.csv
new file mode 100644
index 00000000000..a8a7c5a2414
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge6.csv
@@ -0,0 +1,2 @@
+7
+11
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge7.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge7.csv
new file mode 100644
index 00000000000..8e29be1395c
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge7.csv
@@ -0,0 +1,2 @@
+6,3,1
+11,3,0
\ No newline at end of file
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge8.csv
b/regression-test/data/unique_with_mow_p0/partial_update/merge8.csv
new file mode 100644
index 00000000000..20bbd5c6142
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/merge8.csv
@@ -0,0 +1,2 @@
+7,5
+11,5
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
new file mode 100644
index 00000000000..04e4b07f2dc
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
@@ -0,0 +1,341 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+3 3 3 3
+4 4 4 4
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+
+-- !sql_1_1 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+
+-- !sql_1_2 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !sql_2_1 --
+0 0 0 0
+3 3 30 3
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !sql_2_2 --
+0 0 0 0
+3 3 30 3
+8 8 8 8
+10 \N 999 \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+3 3 3 3
+4 4 4 4
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+
+-- !sql_3_1 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+6 6 6 6 1 0
+7 7 7 7 1 0
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+
+-- !sql_3_2 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+6 6 6 6 1 0
+6 6 6 6 3 1
+7 7 7 7 1 0
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+11 \N 888 \N 3 0
+
+-- !sql_4_1 --
+0 0 0 0
+3 3 30 3
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+1 1 11 1 4 1
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+5 5 50 5 4 1
+6 6 6 6 1 0
+6 6 6 6 3 1
+7 7 7 7 1 0
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+11 \N 888 \N 3 0
+
+-- !sql_4_2 --
+0 0 0 0
+3 3 30 3
+8 8 8 8
+10 \N 999 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+1 1 11 1 4 1
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+5 5 50 5 4 1
+6 6 6 6 1 0
+6 6 6 6 3 1
+7 7 7 7 1 0
+7 7 7 7 5 1
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+11 \N 888 \N 3 0
+11 \N 888 \N 5 1
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+3 3 3 3
+4 4 4 4
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+
+-- !sql_1_1 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+
+-- !sql_1_2 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !sql_2_1 --
+0 0 0 0
+3 3 30 3
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !sql_2_2 --
+0 0 0 0
+3 3 30 3
+8 8 8 8
+10 \N 999 \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+3 3 3 3
+4 4 4 4
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+
+-- !sql_3_1 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+6 6 6 6
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+6 6 6 6 1 0
+7 7 7 7 1 0
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+
+-- !sql_3_2 --
+0 0 0 0
+1 1 10 1
+3 3 30 3
+5 5 5 5
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+6 6 6 6 1 0
+6 6 6 6 3 1
+7 7 7 7 1 0
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+11 \N 888 \N 3 0
+
+-- !sql_4_1 --
+0 0 0 0
+3 3 30 3
+7 7 7 7
+8 8 8 8
+10 \N 999 \N
+11 \N 888 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+1 1 11 1 4 1
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+5 5 50 5 4 1
+6 6 6 6 1 0
+6 6 6 6 3 1
+7 7 7 7 1 0
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+11 \N 888 \N 3 0
+
+-- !sql_4_2 --
+0 0 0 0
+3 3 30 3
+8 8 8 8
+10 \N 999 \N
+
+-- !inspect --
+0 0 0 0 1 0
+1 1 1 1 1 0
+1 1 10 1 2 0
+1 1 11 1 4 1
+2 2 2 2 1 0
+2 2 20 2 2 1
+3 3 3 3 1 0
+3 3 30 3 2 0
+4 4 4 4 1 0
+4 4 40 4 2 1
+5 5 5 5 1 0
+5 5 50 5 4 1
+6 6 6 6 1 0
+6 6 6 6 3 1
+7 7 7 7 1 0
+7 7 7 7 5 1
+8 8 8 8 1 0
+10 \N 999 \N 2 0
+11 \N 888 \N 2 1
+11 \N 888 \N 3 0
+11 \N 888 \N 5 1
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy
new file mode 100644
index 00000000000..ece523838b0
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy
@@ -0,0 +1,185 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_partial_update_merge_type", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ def inspect_rows = { sqlStr ->
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_inspect sqlStr
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+
+ def tableName = "test_partial_update_merge_type"
+ sql """ DROP TABLE IF EXISTS ${tableName} force"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` BIGINT NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int)
+ UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+ sql """insert into ${tableName} select number,number,number,number
from numbers("number"="9");"""
+ sql "sync"
+ qt_sql """select * from ${tableName} order by k;"""
+ // 1.1 merge_type=MERGE, no sequence col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,c2,del'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'MERGE'
+ set 'delete', 'del=1'
+ file 'merge1.csv'
+ time 10000
+ }
+ qt_sql_1_1 """select * from ${tableName} order by k;"""
+ // 1.2 merge_type=MERGE, no sequence col, no value col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,del'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'MERGE'
+ set 'delete', 'del=1'
+ file 'merge5.csv'
+ time 10000
+ }
+ qt_sql_1_2 """select * from ${tableName} order by k;"""
+ // 2.1 merge_type=DELETE, no sequence col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,c1'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'DELETE'
+ file 'merge2.csv'
+ time 10000
+ }
+ qt_sql_2_1 """select * from ${tableName} order by k;"""
+ // 2.2 merge_type=DELETE, no sequence col, no value col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'DELETE'
+ file 'merge6.csv'
+ time 10000
+ }
+ qt_sql_2_2 """select * from ${tableName} order by k;"""
+
+
+ tableName = "test_partial_update_merge_type2"
+ sql """ DROP TABLE IF EXISTS ${tableName} force"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` BIGINT NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int)
+ UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "function_column.sequence_type" = "BIGINT",
+ "store_row_column" = "${use_row_store}"); """
+ sql """insert into ${tableName}(k,c1,c2,c3,__DORIS_SEQUENCE_COL__)
select number,number,number,number,1 from numbers("number"="9");"""
+ qt_sql """select * from ${tableName} order by k;"""
+ // 3.1 merge_type=MERGE, has sequence type col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,c2,seq,del'
+ set 'partial_columns', 'true'
+ set 'function_column.sequence_col', 'seq'
+ set 'merge_type', 'MERGE'
+ set 'delete', 'del=1'
+ file 'merge3.csv'
+ time 10000
+ }
+ qt_sql_3_1 """select * from ${tableName} order by k;"""
+ inspect_rows """select
k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order
by k,__DORIS_SEQUENCE_COL__;"""
+ // 3.2 merge_type=MERGE, has sequence type col, no value col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,seq,del'
+ set 'partial_columns', 'true'
+ set 'function_column.sequence_col', 'seq'
+ set 'merge_type', 'MERGE'
+ set 'delete', 'del=1'
+ file 'merge7.csv'
+ time 10000
+ }
+ qt_sql_3_2 """select * from ${tableName} order by k;"""
+ inspect_rows """select
k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order
by k,__DORIS_SEQUENCE_COL__;"""
+
+ // 4.1 merge_type=DELETE, has sequence type col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,c2,seq'
+ set 'partial_columns', 'true'
+ set 'function_column.sequence_col', 'seq'
+ set 'merge_type', 'DELETE'
+ file 'merge4.csv'
+ time 10000
+ }
+ qt_sql_4_1 """select * from ${tableName} order by k;"""
+ inspect_rows """select
k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order
by k,__DORIS_SEQUENCE_COL__;"""
+ // 4.2 merge_type=DELETE, has sequence type col, no value col
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k,seq'
+ set 'partial_columns', 'true'
+ set 'function_column.sequence_col', 'seq'
+ set 'merge_type', 'DELETE'
+ file 'merge8.csv'
+ time 10000
+ }
+ qt_sql_4_2 """select * from ${tableName} order by k;"""
+ inspect_rows """select
k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order
by k,__DORIS_SEQUENCE_COL__;"""
+
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
