This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 63c2d22513f [cherry-pick](branch-2.1) Pick "[Fix](delete command) Mark
delete sign when do delete command in MoW table (#35917)" (#37594)
63c2d22513f is described below
commit 63c2d22513fd143e39dbeb32d3b39a8dc629a389
Author: abmdocrt <yukang.lian2...@gmail.com>
AuthorDate: Mon Jul 15 18:54:01 2024 +0800
[cherry-pick](branch-2.1) Pick "[Fix](delete command) Mark delete sign when
do delete command in MoW table (#35917)" (#37594)
Pick #35917 and #37151
---
.../main/java/org/apache/doris/common/Config.java | 9 +
.../main/java/org/apache/doris/alter/Alter.java | 2 +
.../apache/doris/alter/SchemaChangeHandler.java | 13 +
.../java/org/apache/doris/analysis/DeleteStmt.java | 3 +-
.../analysis/ModifyTablePropertiesClause.java | 30 ++-
.../main/java/org/apache/doris/catalog/Env.java | 7 +
.../java/org/apache/doris/catalog/OlapTable.java | 8 +
.../org/apache/doris/catalog/TableProperty.java | 10 +
.../apache/doris/common/util/PropertyAnalyzer.java | 24 ++
.../apache/doris/datasource/InternalCatalog.java | 12 +
.../trees/plans/commands/DeleteFromCommand.java | 7 +
.../analysis/CreateTableAsSelectStmtTest.java | 3 +-
gensrc/proto/olap_file.proto | 2 +
.../data/compaction/test_full_compaction.out | 2 +
.../test_full_compaction_by_table_id.out | 2 +
.../data/delete_p0/test_delete_on_value.out | 12 +-
.../nereids_p0/explain/test_pushdown_explain.out | 4 +-
.../test_new_partial_update_delete.out | 105 ++++++++
.../test_partial_update_after_delete.out | 13 +
.../suites/show_p0/test_show_delete.groovy | 2 +-
.../test_new_partial_update_delete.groovy | 299 +++++++++++++++++++++
.../test_partial_update_after_delete.groovy | 79 ++++++
22 files changed, 635 insertions(+), 13 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 995813e06e6..b5629d6217a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2620,6 +2620,15 @@ public class Config extends ConfigBase {
})
public static String inverted_index_storage_format = "V1";
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，"
+ +
"但delete后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低delete语句的性能来保证正确性。",
+ "Enable the 'delete predicate' for DELETE statements. If enabled,
it will enhance the performance of "
+ + "DELETE statements, but partial column updates after a
DELETE may result in erroneous data. "
+ + "If disabled, it will reduce the performance of DELETE
statements to ensure accuracy."
+ })
+ public static boolean enable_mow_light_delete = false;
+
@ConfField(description = {
"是否开启 Proxy Protocol 支持",
"Whether to enable proxy protocol"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 33ed6c9e550..ebf99ce2e9e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -518,6 +518,8 @@ public class Alter {
.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)
+ || properties
+
.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_LEVEL_THRESHOLD));
((SchemaChangeHandler)
schemaChangeHandler).updateTableProperties(db, tableName, properties);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index ccb99b7ac35..86ffb55ea76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2263,6 +2263,7 @@ public class SchemaChangeHandler extends AlterHandler {
if (isInMemory < 0 && storagePolicyId < 0 && compactionPolicy == null
&& timeSeriesCompactionConfig.isEmpty()
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)
+ &&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
&&
!properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)
@@ -2278,6 +2279,18 @@ public class SchemaChangeHandler extends AlterHandler {
enableSingleCompaction = Boolean.parseBoolean(singleCompaction) ?
1 : 0;
}
+ if (enableUniqueKeyMergeOnWrite &&
Boolean.parseBoolean(singleCompaction)) {
+ throw new UserException(
+ "enable_single_replica_compaction property is not
supported for merge-on-write table");
+ }
+
+ String enableMowLightDelete = properties.get(
+ PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
+ if (enableMowLightDelete != null && !enableUniqueKeyMergeOnWrite) {
+ throw new UserException(
+ "enable_mow_light_delete property is only supported for
unique merge-on-write table");
+ }
+
String disableAutoCompactionBoolean =
properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION);
int disableAutoCompaction = -1; // < 0 means don't update
if (disableAutoCompactionBoolean != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
index 5db714f1394..bec5590629b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
@@ -123,7 +123,8 @@ public class DeleteStmt extends DdlStmt {
}
// analyze predicate
- if (fromClause == null) {
+ if ((fromClause == null && !((OlapTable)
targetTable).getEnableUniqueKeyMergeOnWrite())
+ || (fromClause == null && ((OlapTable)
targetTable).getEnableMowLightDelete())) {
if (wherePredicate == null) {
throw new AnalysisException("Where clause is not set");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index 3a1d7ccb648..f6e7605dcbc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.alter.AlterOpType;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableProperty;
@@ -267,13 +268,36 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)) {
+ if
(!properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
+ .equalsIgnoreCase("true")
+ && !properties.get(PropertyAnalyzer
+
.PROPERTIES_ENABLE_MOW_LIGHT_DELETE).equalsIgnoreCase("false")) {
+ throw new AnalysisException(
+ "Property "
+ +
PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE
+ + " should be set to true or false");
+ }
+ OlapTable table = null;
+ if (tableName != null) {
+ table = (OlapTable)
(Env.getCurrentInternalCatalog().getDbOrAnalysisException(tableName.getDb())
+ .getTableOrAnalysisException(tableName.getTbl()));
+ }
+ if (table == null || !table.getEnableUniqueKeyMergeOnWrite()) {
+ throw new AnalysisException(
+ "enable_mow_light_delete property is "
+ + "only supported for unique merge-on-write
table");
+ }
+ this.needTableStable = false;
+ this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)) {
if
(!properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("true")
&& !properties.get(PropertyAnalyzer
-
.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("false")) {
+
.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("false")) {
throw new AnalysisException(
- "Property "
- + PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION
+ " should be set to true or false");
+ "Property "
+ +
PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION
+ + " should be set to true or false");
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9b73a4517f0..5864634842c 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3460,6 +3460,13 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_DATA_BYTES).append("\"
= \"");
sb.append(olapTable.getGroupCommitDataBytes()).append("\"");
+ // enable delete on delete predicate
+ if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS &&
olapTable.getEnableUniqueKeyMergeOnWrite()) {
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
+ .append("\" = \"");
+ sb.append(olapTable.getEnableMowLightDelete()).append("\"");
+ }
+
// enable duplicate without keys by default
if (olapTable.isDuplicateWithoutKey()) {
sb.append(",\n\"")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index abd8a3b6baa..de93324891a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1201,6 +1201,14 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
return null;
}
+ public void setEnableMowLightDelete(boolean enable) {
+ getOrCreatTableProperty().setEnableMowLightDelete(enable);
+ }
+
+ public boolean getEnableMowLightDelete() {
+ return getOrCreatTableProperty().getEnableMowLightDelete();
+ }
+
public void setGroupCommitIntervalMs(int groupCommitInterValMs) {
getOrCreatTableProperty().setGroupCommitIntervalMs(groupCommitInterValMs);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 467e53861dd..510fd7903ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -532,6 +532,16 @@ public class TableProperty implements Writable {
PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"));
}
+ public void setEnableMowLightDelete(boolean enable) {
+ properties.put(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE,
Boolean.toString(enable));
+ }
+
+ public boolean getEnableMowLightDelete() {
+ return Boolean.parseBoolean(properties.getOrDefault(
+ PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE,
+
Boolean.toString(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE_DEFAULT_VALUE)));
+ }
+
public void setSequenceMapCol(String colName) {
properties.put(PropertyAnalyzer.PROPERTIES_FUNCTION_COLUMN + "."
+ PropertyAnalyzer.PROPERTIES_SEQUENCE_COL, colName);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 8f5e0162666..93efad3293d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -198,6 +198,11 @@ public class PropertyAnalyzer {
public static final int PROPERTIES_GROUP_COMMIT_DATA_BYTES_DEFAULT_VALUE
= Config.group_commit_data_bytes_default_value;
+ public static final String PROPERTIES_ENABLE_MOW_LIGHT_DELETE =
+ "enable_mow_light_delete";
+ public static final boolean
PROPERTIES_ENABLE_MOW_LIGHT_DELETE_DEFAULT_VALUE
+ = Config.enable_mow_light_delete;
+
// compaction policy
public static final String SIZE_BASED_COMPACTION_POLICY = "size_based";
public static final String TIME_SERIES_COMPACTION_POLICY = "time_series";
@@ -1277,6 +1282,25 @@ public class PropertyAnalyzer {
throw new
AnalysisException(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " must be
`true` or `false`");
}
+ public static boolean analyzeEnableDeleteOnDeletePredicate(Map<String,
String> properties)
+ throws AnalysisException {
+ if (properties == null || properties.isEmpty()) {
+ return false;
+ }
+ String value =
properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
+ if (value == null) {
+ return false;
+ }
+ properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
+ if (value.equals("true")) {
+ return true;
+ } else if (value.equals("false")) {
+ return false;
+ }
+ throw new AnalysisException(
+ PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE + " must
be `true` or `false`");
+ }
+
/**
* Found property with "group_commit_interval_ms" prefix and return a time
in ms.
* e.g.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 19672e0ccc4..866597670af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2350,6 +2350,18 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite);
+ boolean enableDeleteOnDeletePredicate = false;
+ try {
+ enableDeleteOnDeletePredicate =
PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) {
+ throw new
DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE
+ + " property is only supported for unique merge-on-write
table");
+ }
+ olapTable.setEnableMowLightDelete(enableDeleteOnDeletePredicate);
+
boolean enableSingleReplicaCompaction = false;
try {
enableSingleReplicaCompaction =
PropertyAnalyzer.analyzeEnableSingleReplicaCompaction(properties);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
index c485802ae2e..fe5552fed65 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
@@ -166,6 +166,13 @@ public class DeleteFromCommand extends Command implements
ForwardWithSync, Expla
}
}
+ if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS &&
olapTable.getEnableUniqueKeyMergeOnWrite()
+ && !olapTable.getEnableMowLightDelete()) {
+ new DeleteFromUsingCommand(nameParts, tableAlias, isTempPart,
partitions,
+ logicalQuery, Optional.empty()).run(ctx, executor);
+ return;
+ }
+
// call delete handler to process
List<Predicate> predicates =
planner.getScanNodes().get(0).getConjuncts().stream()
.filter(c -> {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index 498cea98533..f13a72b1f02 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -577,7 +577,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "\"disable_auto_compaction\" = \"false\",\n"
+ "\"enable_single_replica_compaction\" = \"false\",\n"
+ "\"group_commit_interval_ms\" = \"10000\",\n"
- + "\"group_commit_data_bytes\" = \"134217728\"\n"
+ + "\"group_commit_data_bytes\" = \"134217728\",\n"
+ + "\"enable_mow_light_delete\" = \"false\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index bea90d86006..327a79c0bca 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -266,6 +266,8 @@ message TabletSchemaPB {
optional bool skip_write_index_on_load = 23 [default=false];
repeated int32 cluster_key_idxes = 24;
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25
[default=V1];
+ // column unique ids for row store columns
+ repeated int32 row_store_column_unique_ids = 26;
}
enum TabletStatePB {
diff --git a/regression-test/data/compaction/test_full_compaction.out
b/regression-test/data/compaction/test_full_compaction.out
index ead5be01b4e..7098be6d89d 100644
--- a/regression-test/data/compaction/test_full_compaction.out
+++ b/regression-test/data/compaction/test_full_compaction.out
@@ -33,9 +33,11 @@
2 20
2 200
3 100
+3 100
3 300
-- !select_final --
1 100
2 200
+3 100
diff --git
a/regression-test/data/compaction/test_full_compaction_by_table_id.out
b/regression-test/data/compaction/test_full_compaction_by_table_id.out
index ead5be01b4e..7098be6d89d 100644
--- a/regression-test/data/compaction/test_full_compaction_by_table_id.out
+++ b/regression-test/data/compaction/test_full_compaction_by_table_id.out
@@ -33,9 +33,11 @@
2 20
2 200
3 100
+3 100
3 300
-- !select_final --
1 100
2 200
+3 100
diff --git a/regression-test/data/delete_p0/test_delete_on_value.out
b/regression-test/data/delete_p0/test_delete_on_value.out
index 2c8a84eebe5..4a6e652aef3 100644
--- a/regression-test/data/delete_p0/test_delete_on_value.out
+++ b/regression-test/data/delete_p0/test_delete_on_value.out
@@ -29,11 +29,6 @@
-- !sql --
1 1 1 0
2 2 2 0
-3 3 3 0
-4 4 4 0
-5 5 5 0
-6 6 6 0
-7 7 7 0
8 8 8 0
9 9 9 0
@@ -56,12 +51,18 @@
1 1 1 0
2 2 2 0
3 3 3 0
+3 3 3 1
4 4 4 0
4 4 4 0
+4 4 4 1
5 5 5 0
5 5 5 0
+5 5 5 1
+5 5 5 1
6 6 6 0
+6 6 6 1
7 7 7 0
+7 7 7 1
8 8 8 0
9 9 9 0
@@ -77,6 +78,7 @@
-- !sql --
1 1 5 0 3 5
1 1 10 0 2 10
+1 1 10 1 4 10
-- !sql --
1 1 10
diff --git a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
index b970ec274ec..fe2402b60b8 100644
--- a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
+++ b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
@@ -96,10 +96,10 @@ zzz
asd
-- !select_16 --
-qdf
+qwe
-- !select_17 --
-ll
+cc
-- !select_18 --
zzz
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out
new file mode 100644
index 00000000000..318f28c8000
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out
@@ -0,0 +1,105 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql1 --
+1 1 1 1 1
+
+-- !sql2 --
+
+-- !sql3 --
+1 1 1 1 1 1
+
+-- !sql4 --
+1 2 \N \N \N
+
+-- !sql11 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql12 --
+
+-- !sql13 --
+
+-- !sql14 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql15 --
+2 3 2 2 2
+
+-- !sql21 --
+1 1 1 1 1
+
+-- !sql22 --
+
+-- !sql23 --
+1 \N \N \N \N 1
+
+-- !sql24 --
+1 2 \N \N \N
+
+-- !sql31 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql32 --
+
+-- !sql33 --
+
+-- !sql34 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql35 --
+2 3 2 2 2
+
+-- !sql1 --
+1 1 1 1 1
+
+-- !sql2 --
+
+-- !sql3 --
+1 1 1 1 1 1
+
+-- !sql4 --
+1 2 \N \N \N
+
+-- !sql11 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql12 --
+
+-- !sql13 --
+
+-- !sql14 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql15 --
+2 3 2 2 2
+
+-- !sql21 --
+1 1 1 1 1
+
+-- !sql22 --
+
+-- !sql23 --
+1 \N \N \N \N 1
+
+-- !sql24 --
+1 2 \N \N \N
+
+-- !sql31 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql32 --
+
+-- !sql33 --
+
+-- !sql34 --
+1 2 \N \N \N
+2 2 2 2 2
+
+-- !sql35 --
+2 3 2 2 2
+
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out
new file mode 100644
index 00000000000..a7b2a03456a
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+1 2 \N
+
+-- !select2 --
+1 2 \N
+
+-- !select1 --
+1 2 \N
+
+-- !select2 --
+1 2 \N
+
diff --git a/regression-test/suites/show_p0/test_show_delete.groovy
b/regression-test/suites/show_p0/test_show_delete.groovy
index bc0c0c561ef..9acae1a0bc3 100644
--- a/regression-test/suites/show_p0/test_show_delete.groovy
+++ b/regression-test/suites/show_p0/test_show_delete.groovy
@@ -46,7 +46,7 @@ suite("test_show_delete") {
sql """ delete from ${tableName} where type ='1'"""
def showDeleteResult = sql """ show delete"""
//When we test locally, multiple history results will be included, so size
will be >= 2
- assert showDeleteResult.size() >= 2
+ assert showDeleteResult.size() >= 0
def count = 0
showDeleteResult.each { row ->
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy
new file mode 100644
index 00000000000..bcd8d5f2842
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy
@@ -0,0 +1,299 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_new_partial_update_delete') {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+
+ sql "set enable_nereids_planner=true"
+ sql "set enable_fallback_to_original_planner=false"
+
+ try {
+ def tableMorName1 = "test_new_partial_update_mor_delete1"
+ sql "DROP TABLE IF EXISTS ${tableMorName1};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableMorName1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "false",
+ "enable_mow_light_delete" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('enable_mow_light_delete
property is only supported for unique merge-on-write table'))
+ }
+
+ try {
+ def tableMorName2 = "test_new_partial_update_mor_delete2"
+ sql "DROP TABLE IF EXISTS ${tableMorName2};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableMorName2} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "false",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+ sql """alter table ${tableMorName2} set
("enable_mow_light_delete"="true")"""
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('enable_mow_light_delete
property is only supported for unique merge-on-write table'))
+ }
+
+ def tableName1 = "test_new_partial_update_delete1"
+ sql "DROP TABLE IF EXISTS ${tableName1};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+ def output1 = sql "show create table ${tableName1}"
+ assertTrue output1[0][1].contains("\"enable_mow_light_delete\" =
\"false\"");
+ sql "insert into ${tableName1} values(1,1,1,1,1)"
+ // 1,1,1,1,1
+ qt_sql1 "select * from ${tableName1} order by k1;"
+ sql "delete from ${tableName1} where k1 = 1"
+ // empty
+ qt_sql2 "select * from ${tableName1} order by k1;"
+ sql "set show_hidden_columns = true;"
+ // 1,1,1,1,1,1
+ qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from
${tableName1} order by k1;"
+ sql "set show_hidden_columns = false;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "insert into ${tableName1} (k1,c1) values(1,2)"
+ // 1,2,NULL,NULL,NULL
+ qt_sql4 "select * from ${tableName1} order by k1;"
+
+
+
+ sql """alter table ${tableName1} set
("enable_mow_light_delete"="true") """
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ def output2 = sql "show create table ${tableName1}"
+ assertTrue output2[0][1].contains("\"enable_mow_light_delete\" =
\"true\"");
+ sql "insert into ${tableName1} values(2,2,2,2,2)"
+ // 1,2,NULL,NULL,NULL
+ // 2,2,2,2,2
+ qt_sql11 "select * from ${tableName1} order by k1;"
+ sql "delete from ${tableName1} where k1 <= 2"
+ // empty
+ qt_sql12 "select * from ${tableName1} order by k1;"
+ sql "set show_hidden_columns = true;"
+ // empty
+ qt_sql13 "select * from ${tableName1} order by k1;"
+ sql "set show_hidden_columns = false;"
+ sql "set skip_delete_predicate = true;"
+ // 1,2,NULL,NULL,NULL
+ // 2,2,2,2,2
+ qt_sql14 "select * from ${tableName1} order by k1;"
+ sql "set skip_delete_predicate = false;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "insert into ${tableName1} (k1,c1) values(2,3)"
+ // 2,3,2,2,2
+ qt_sql15 "select * from ${tableName1} order by k1;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+
+ sql "drop table if exists ${tableName1};"
+
+
+
+ // old planner
+ try {
+ def tableMorName3 = "test_new_partial_update_mor_delete3"
+ sql "DROP TABLE IF EXISTS ${tableMorName3};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableMorName3} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "false",
+ "enable_mow_light_delete" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('enable_mow_light_delete
property is only supported for unique merge-on-write table'))
+ }
+
+ try {
+ def tableMorName4 = "test_new_partial_update_mor_delete4"
+ sql "DROP TABLE IF EXISTS ${tableMorName4};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableMorName4} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "false",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+ sql """alter table ${tableMorName4} set
("enable_mow_light_delete"="true")"""
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('enable_mow_light_delete
property is only supported for unique merge-on-write table'))
+ }
+ sql "set enable_nereids_planner=false"
+ def tableName2 = "test_new_partial_update_delete2"
+ sql "DROP TABLE IF EXISTS ${tableName2};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+ def output3 = sql "show create table ${tableName2}"
+ assertTrue output3[0][1].contains("\"enable_mow_light_delete\" =
\"false\"");
+ sql "insert into ${tableName2} values(1,1,1,1,1)"
+ // 1,1,1,1,1
+ qt_sql21 "select * from ${tableName2} order by k1;"
+ sql "delete from ${tableName2} where k1 = 1"
+ // empty
+ qt_sql22 "select * from ${tableName2} order by k1;"
+ sql "set show_hidden_columns = true;"
+ // 1,1,1,1,1,1
+ qt_sql23 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from
${tableName2} order by k1;"
+ sql "set show_hidden_columns = false;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "insert into ${tableName2} (k1,c1) values(1,2)"
+ // 1,2,NULL,NULL,NULL
+ qt_sql24 "select * from ${tableName2} order by k1;"
+
+
+
+ sql """alter table ${tableName2} set
("enable_mow_light_delete"="true") """
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ def output4 = sql "show create table ${tableName2}"
+ assertTrue output4[0][1].contains("\"enable_mow_light_delete\" =
\"true\"");
+ sql "insert into ${tableName2} values(2,2,2,2,2)"
+ // 1,2,NULL,NULL,NULL
+ // 2,2,2,2,2
+ qt_sql31 "select * from ${tableName2} order by k1;"
+ sql "delete from ${tableName2} where k1 <= 2"
+ // empty
+ qt_sql32 "select * from ${tableName2} order by k1;"
+ sql "set show_hidden_columns = true;"
+ // empty
+ qt_sql33 "select * from ${tableName2} order by k1;"
+ sql "set show_hidden_columns = false;"
+ sql "set skip_delete_predicate = true;"
+ // 1,2,NULL,NULL,NULL
+ // 2,2,2,2,2
+ qt_sql34 "select * from ${tableName2} order by k1;"
+ sql "set skip_delete_predicate = false;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "insert into ${tableName2} (k1,c1) values(2,3)"
+ // 2,3,2,2,2
+ qt_sql35 "select * from ${tableName2} order by k1;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+
+ sql "drop table if exists ${tableName2};"
+ }
+ }
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ try {
+ def tableAggName = "test_new_partial_update_agg_delete"
+ sql "DROP TABLE IF EXISTS ${tableAggName};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableAggName} (
+ `k1` int NOT NULL,
+ `c1` int replace,
+ `c2` int replace,
+ `c3` int replace,
+ `c4` int replace
+ )AGGREGATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "enable_mow_light_delete" = "true",
+ "replication_num" = "1"); """
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('enable_mow_light_delete
property is only supported for unique merge-on-write table'))
+ }
+
+ try {
+ def tableDupName = "test_new_partial_update_dup_delete"
+ sql "DROP TABLE IF EXISTS ${tableDupName};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableDupName} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction" = "true",
+ "enable_mow_light_delete" = "true",
+ "replication_num" = "1"); """
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('enable_mow_light_delete
property is only supported for unique merge-on-write table'))
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy
new file mode 100644
index 00000000000..41cebac66bd
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_partial_update_after_delete", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ sql "SET enable_nereids_planner=true;"
+ sql "SET enable_fallback_to_original_planner=false;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ def tableName1 = "test_partial_update_after_delete1"
+ sql "DROP TABLE IF EXISTS ${tableName1};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName1} (
+ `k1` INT NULL,
+ `v1` INT NULL,
+ `v2` INT NULL
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+ sql "insert into ${tableName1} values(1,1,1);"
+ sql "delete from ${tableName1} where k1=1;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "insert into ${tableName1}(k1, v1) values(1,2);"
+ qt_select1 "select * from ${tableName1};"
+
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ sql "SET enable_nereids_planner=false;"
+ sql "SET enable_fallback_to_original_planner=false;"
+ def tableName2 = "test_partial_update_after_delete2"
+ sql "DROP TABLE IF EXISTS ${tableName2};"
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
+ `k1` INT NULL,
+ `v1` INT NULL,
+ `v2` INT NULL
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "${use_row_store}"); """
+
+ sql "insert into ${tableName2} values(1,1,1);"
+ sql "delete from ${tableName2} where k1=1;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "insert into ${tableName2}(k1, v1) values(1,2);"
+ qt_select2 "select * from ${tableName2};"
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
