This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 74b85d6bffe branch-2.1-pick: [compaction](config) Add a config to
control whether to prune rows with delete sign=1 in base compaction (#48241)
(#48620)
74b85d6bffe is described below
commit 74b85d6bffe49e9390e2ec26e92e7c366fd8c61d
Author: bobhan1 <bao...@selectdb.com>
AuthorDate: Wed Mar 5 10:10:58 2025 +0800
branch-2.1-pick: [compaction](config) Add a config to control whether to
prune rows with delete sign=1 in base compaction (#48241) (#48620)
pick https://github.com/apache/doris/pull/48241
---
be/src/common/config.cpp | 4 +
be/src/common/config.h | 4 +
be/src/olap/tablet_reader.cpp | 3 +-
.../compaction/test_config_prune_delete_sign.out | Bin 0 -> 163 bytes
.../test_config_prune_delete_sign.groovy | 89 +++++++++++++++++++++
5 files changed, 99 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 1f81e646045..987f09f3433 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1360,6 +1360,10 @@
DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false");
DEFINE_mInt32(compaction_num_per_round, "1");
+// whether to prune rows with delete sign = 1 in base compaction
+// ATTN: this config is only for test
+DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index afad2cac7e7..b11534a6e40 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1432,6 +1432,10 @@ DECLARE_mInt32(compaction_num_per_round);
// Enable sleep 5s between delete cumulative compaction.
DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
+// whether to prune rows with delete sign = 1 in base compaction
+// ATTN: this config is only for test
+DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 51b2384bc90..0a9f7119ffe 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -628,7 +628,8 @@ Status TabletReader::_init_delete_condition(const
ReaderParams& read_params) {
// Only BASE_COMPACTION and COLD_DATA_COMPACTION and CUMULATIVE_COMPACTION
need set filter_delete = true
// other reader type:
// QUERY will filter the row in query layer to keep right result use where
clause.
- _filter_delete = (read_params.reader_type ==
ReaderType::READER_BASE_COMPACTION ||
+ _filter_delete = ((read_params.reader_type ==
ReaderType::READER_BASE_COMPACTION &&
+ config::enable_prune_delete_sign_when_base_compaction)
||
read_params.reader_type ==
ReaderType::READER_COLD_DATA_COMPACTION ||
((read_params.reader_type ==
ReaderType::READER_CUMULATIVE_COMPACTION &&
config::enable_delete_when_cumu_compaction)) ||
diff --git a/regression-test/data/compaction/test_config_prune_delete_sign.out
b/regression-test/data/compaction/test_config_prune_delete_sign.out
new file mode 100644
index 00000000000..9bbb4251318
Binary files /dev/null and
b/regression-test/data/compaction/test_config_prune_delete_sign.out differ
diff --git
a/regression-test/suites/compaction/test_config_prune_delete_sign.groovy
b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy
new file mode 100644
index 00000000000..06c56ce6405
--- /dev/null
+++ b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_config_prune_delete_sign", "nonConcurrent") {
+
+ def inspectRows = { sqlStr ->
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_inspect sqlStr
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+ def custoBeConfig = [
+ enable_prune_delete_sign_when_base_compaction : false
+ ]
+
+ setBeConfigTemporary(custoBeConfig) {
+ def table1 = "test_config_prune_delete_sign"
+ sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+ sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_mow_light_delete" = "false",
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1"); """
+
+ def getDeleteSignCnt = {
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_del_cnt "select count() from ${table1} where
__DORIS_DELETE_SIGN__=1;"
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+ (1..30).each {
+ sql "insert into ${table1} values($it,$it,$it);"
+ }
+ trigger_and_wait_compaction(table1, "cumulative")
+
+ sql "delete from ${table1} where k1<=20;"
+ sql "sync;"
+ qt_sql "select count() from ${table1};"
+ getDeleteSignCnt()
+
+ (31..60).each {
+ sql "insert into ${table1} values($it,$it,$it);"
+ }
+ trigger_and_wait_compaction(table1, "cumulative")
+
+ trigger_and_wait_compaction(table1, "base")
+ qt_sql "select count() from ${table1};"
+ getDeleteSignCnt()
+
+ def tablets = sql_return_maparray """ show tablets from ${table1}; """
+ logger.info("tablets: ${tablets}")
+ assert 1 == tablets.size()
+ String compactionUrl = tablets[0]["CompactionStatus"]
+ def (code, out, err) = curl("GET", compactionUrl)
+ logger.info("Show tablets status: code=" + code + ", out=" + out + ",
err=" + err)
+ assert code == 0
+ def tabletJson = parseJson(out.trim())
+ assert tabletJson.rowsets.size() == 1
+ assert tabletJson.rowsets[0].contains("[0-62]")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
