This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new eb7ac1688ad [Opt](mow) Calc delete bitmaps between segments
concurrently in commit phase (#50106)
eb7ac1688ad is described below
commit eb7ac1688adf429d6f8ae78c2242484e69aa6907
Author: bobhan1 <bao...@selectdb.com>
AuthorDate: Mon Apr 28 12:22:41 2025 +0800
[Opt](mow) Calc delete bitmaps between segments concurrently in commit
phase (#50106)
### What problem does this PR solve?
This PR make the calculation of delete bitmaps between segments in
commit phase and the calculation of delete bitmaps of current load's
rowsets with incremental historical data in commit phase concurrently
for all tablets under one load channel. We use the initial load time(no
historical data) of large dataset which can produce multi segments to
evaluate the effectiveness of this PR.
test:
||210GB origin data files, 2 varchar(2000) as keys, cloud mode, 1 * FE
4C 16GB， 2 * BE 16C 128GB，PL2 ESSD*2, before|210GB origin data files, 2
varchar(2000) as keys, cloud mode, 1 * FE 4C 16GB， 2 * BE 16C 128GB，PL2
ESSD*2, with this PR|
|---|---|---|
|total time(s)|9 min 40 s|6 min 2 s|
|one load channel from beginning to last memtable flush|3 min 58 s|4 min
29 s|
|one load channel from last memtable flush to load channel close|5 min
40 s|1 min 25 s|
note that when calculating delete bitmaps between segments concurrently
in commit phase, it requires high read iops and throughput
with this pr:
without this pr:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [x] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/calc_delete_bitmap_executor.cpp | 23 +++++++++++++++++++++++
be/src/olap/calc_delete_bitmap_executor.h | 6 ++++++
be/src/olap/rowset_builder.cpp | 9 +++++++--
5 files changed, 40 insertions(+), 2 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 35b7f5f3463..61a72d25e5e 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1490,6 +1490,8 @@ DEFINE_mInt32(load_trigger_compaction_version_percent,
"66");
DEFINE_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
DEFINE_mBool(enable_compaction_pause_on_high_memory, "true");
+DEFINE_mBool(enable_calc_delete_bitmap_between_segments_concurrently, "false");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index fb0153c47e7..918d18bf3ef 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1564,6 +1564,8 @@ DECLARE_mInt32(load_trigger_compaction_version_percent);
DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
DECLARE_mBool(enable_compaction_pause_on_high_memory);
+DECLARE_mBool(enable_calc_delete_bitmap_between_segments_concurrently);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp
b/be/src/olap/calc_delete_bitmap_executor.cpp
index 57960693ea2..dfb91d09b25 100644
--- a/be/src/olap/calc_delete_bitmap_executor.cpp
+++ b/be/src/olap/calc_delete_bitmap_executor.cpp
@@ -60,6 +60,29 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet,
RowsetSharedPtr cur_
});
}
+Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetId rowset_id,
+ const
std::vector<segment_v2::SegmentSharedPtr>& segments,
+ DeleteBitmapPtr delete_bitmap) {
+ {
+ std::shared_lock rlock(_lock);
+ RETURN_IF_ERROR(_status);
+ _resource_ctx = thread_context()->resource_ctx();
+ }
+ return _thread_token->submit_func([=, this]() {
+ SCOPED_ATTACH_TASK(_resource_ctx);
+ auto st = tablet->calc_delete_bitmap_between_segments(rowset_id,
segments, delete_bitmap);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to calc delete bitmap between segments,
tablet_id: "
+ << tablet->tablet_id() << " rowset: " << rowset_id
+ << " segments num: " << segments.size() << " error: "
<< st;
+ std::lock_guard wlock(_lock);
+ if (_status.ok()) {
+ _status = st;
+ }
+ }
+ });
+}
+
Status CalcDeleteBitmapToken::wait() {
_thread_token->wait();
// all tasks complete here, don't need lock;
diff --git a/be/src/olap/calc_delete_bitmap_executor.h
b/be/src/olap/calc_delete_bitmap_executor.h
index 708e7fd21bf..9d42ec6a178 100644
--- a/be/src/olap/calc_delete_bitmap_executor.h
+++ b/be/src/olap/calc_delete_bitmap_executor.h
@@ -49,12 +49,18 @@ public:
explicit CalcDeleteBitmapToken(std::unique_ptr<ThreadPoolToken>
thread_token)
: _thread_token(std::move(thread_token)), _status(Status::OK()) {}
+ // calculate delete bitmap of `cur_segment` to historical `target_rowsets`
Status submit(BaseTabletSPtr tablet, RowsetSharedPtr cur_rowset,
const segment_v2::SegmentSharedPtr& cur_segment,
const std::vector<RowsetSharedPtr>& target_rowsets, int64_t
end_version,
DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
DeleteBitmapPtr tablet_delete_bitmap);
+ // calculate delete bitmap between `segments`
+ Status submit(BaseTabletSPtr tablet, RowsetId rowset_id,
+ const std::vector<segment_v2::SegmentSharedPtr>& segments,
+ DeleteBitmapPtr delete_bitmap);
+
// wait all tasks in token to be completed.
Status wait();
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 41953baba3d..235ef837169 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -278,8 +278,13 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task()
{
RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
if (segments.size() > 1) {
// calculate delete bitmap between segments
-
RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments(_rowset->rowset_id(),
segments,
-
_delete_bitmap));
+ if (config::enable_calc_delete_bitmap_between_segments_concurrently) {
+ RETURN_IF_ERROR(_calc_delete_bitmap_token->submit(_tablet,
_rowset->rowset_id(),
+ segments,
_delete_bitmap));
+ } else {
+
RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments(_rowset->rowset_id(),
+
segments, _delete_bitmap));
+ }
}
// tablet is under alter process. The delete bitmap will be calculated
after conversion.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
