(doris) branch master updated: [performance](mow) async calc delete bitmap in add_segment (#48156)

[liaoxin]

This is an automated email from the ASF dual-hosted git repository.

liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 87c6a003cea [performance](mow) async calc delete bitmap in add_segment 
(#48156)
87c6a003cea is described below

commit 87c6a003cea75a5e5a0376915f98e57ed49bd4cb
Author: Kaijie Chen <chenkai...@selectdb.com>
AuthorDate: Mon Mar 3 22:07:48 2025 +0800

    [performance](mow) async calc delete bitmap in add_segment (#48156)
    
    Implement async calculation of delete bitmap in
    `RowsetWriter::add_segment()` to reduce wait in segment flush threads.
    A 100% load performance improvement was observed in
    memtable_on_sink_node + mow cases.
---
 be/src/olap/rowset/beta_rowset_writer.cpp | 13 ++++++++++++-
 be/src/olap/rowset/beta_rowset_writer.h   |  1 +
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp 
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 8093a9d3bca..d4f10b8078a 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -277,6 +277,10 @@ BetaRowsetWriter::~BetaRowsetWriter() {
      * is cancelled, the objects involved in the job should be preserved 
during segcompaction to
      * avoid crashs for memory issues. */
     WARN_IF_ERROR(_wait_flying_segcompaction(), "segment compaction failed");
+
+    if (_calc_delete_bitmap_token != nullptr) {
+        _calc_delete_bitmap_token->cancel();
+    }
 }
 
 Status BaseBetaRowsetWriter::init(const RowsetWriterContext& 
rowset_writer_context) {
@@ -330,7 +334,8 @@ Status 
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
     OlapStopWatch watch;
     RETURN_IF_ERROR(BaseTablet::calc_delete_bitmap(
             _context.tablet, rowset_ptr, segments, specified_rowsets,
-            _context.mow_context->delete_bitmap, 
_context.mow_context->max_version, nullptr));
+            _context.mow_context->delete_bitmap, 
_context.mow_context->max_version,
+            _calc_delete_bitmap_token.get()));
     size_t total_rows = std::accumulate(
             segments.begin(), segments.end(), 0,
             [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum 
+= s->num_rows(); });
@@ -348,6 +353,9 @@ Status BetaRowsetWriter::init(const RowsetWriterContext& 
rowset_writer_context)
     if (_segcompaction_worker) {
         _segcompaction_worker->init_mem_tracker(rowset_writer_context);
     }
+    if (_context.mow_context != nullptr) {
+        _calc_delete_bitmap_token = 
_engine.calc_delete_bitmap_executor()->create_token();
+    }
     return Status::OK();
 }
 
@@ -801,6 +809,9 @@ Status BetaRowsetWriter::_close_file_writers() {
 }
 
 Status BetaRowsetWriter::build(RowsetSharedPtr& rowset) {
+    if (_calc_delete_bitmap_token != nullptr) {
+        RETURN_IF_ERROR(_calc_delete_bitmap_token->wait());
+    }
     RETURN_IF_ERROR(_close_file_writers());
     const auto total_segment_num = _num_segment - _segcompacted_point + 1 + 
_num_segcompacted;
     
RETURN_NOT_OK_STATUS_WITH_WARN(_check_segment_number_limit(total_segment_num),
diff --git a/be/src/olap/rowset/beta_rowset_writer.h 
b/be/src/olap/rowset/beta_rowset_writer.h
index a69d1063a55..c4561b317b7 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -257,6 +257,7 @@ protected:
     fmt::memory_buffer vlog_buffer;
 
     std::shared_ptr<MowContext> _mow_context;
+    std::unique_ptr<CalcDeleteBitmapToken> _calc_delete_bitmap_token;
 
     int64_t _delete_bitmap_ns = 0;
     int64_t _segment_writer_ns = 0;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org