This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 51f64494b4f branch-3.0: [Fix](mow) Fix mow coreudmp in
`BaseTablet::get_rowset_by_ids()` (#55539) (#55639)
51f64494b4f is described below
commit 51f64494b4f7ff2569af668fabb310b054b649d3
Author: bobhan1 <[email protected]>
AuthorDate: Thu Sep 4 15:27:40 2025 +0800
branch-3.0: [Fix](mow) Fix mow coreudmp in
`BaseTablet::get_rowset_by_ids()` (#55539) (#55639)
pick https://github.com/apache/doris/pull/55539
---
be/src/cloud/cloud_rowset_builder.cpp | 4 ++--
be/src/olap/olap_common.h | 6 +++---
be/src/olap/rowset/beta_rowset_writer.cpp | 5 +++--
be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +-
be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp | 2 +-
be/src/olap/rowset_builder.cpp | 14 +++++++-------
be/src/olap/rowset_builder.h | 2 +-
be/test/olap/segcompaction_mow_test.cpp | 8 ++++----
8 files changed, 22 insertions(+), 21 deletions(-)
diff --git a/be/src/cloud/cloud_rowset_builder.cpp
b/be/src/cloud/cloud_rowset_builder.cpp
index b495ae953ca..3da6a55aa44 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -126,7 +126,7 @@ Status CloudRowsetBuilder::set_txn_related_delete_bitmap() {
if (_tablet->enable_unique_key_merge_on_write()) {
if (config::enable_merge_on_write_correctness_check &&
_rowset->num_rows() != 0) {
auto st = _tablet->check_delete_bitmap_correctness(
- _delete_bitmap, _rowset->end_version() - 1, _req.txn_id,
_rowset_ids);
+ _delete_bitmap, _rowset->end_version() - 1, _req.txn_id,
*_rowset_ids);
if (!st.ok()) {
LOG(WARNING) << fmt::format(
"[tablet_id:{}][txn_id:{}][load_id:{}][partition_id:{}] "
@@ -137,7 +137,7 @@ Status CloudRowsetBuilder::set_txn_related_delete_bitmap() {
}
}
_engine.txn_delete_bitmap_cache().set_tablet_txn_info(
- _req.txn_id, _tablet->tablet_id(), _delete_bitmap,
_rowset_ids, _rowset,
+ _req.txn_id, _tablet->tablet_id(), _delete_bitmap,
*_rowset_ids, _rowset,
_req.txn_expiration, _partial_update_info);
}
return Status::OK();
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index f792e3ac6f9..264250636da 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -540,16 +540,16 @@ inline RowsetId extract_rowset_id(std::string_view
filename) {
class DeleteBitmap;
// merge on write context
struct MowContext {
- MowContext(int64_t version, int64_t txnid, const RowsetIdUnorderedSet& ids,
+ MowContext(int64_t version, int64_t txnid,
std::shared_ptr<RowsetIdUnorderedSet> ids,
std::vector<RowsetSharedPtr> rowset_ptrs,
std::shared_ptr<DeleteBitmap> db)
: max_version(version),
txn_id(txnid),
- rowset_ids(ids),
+ rowset_ids(std::move(ids)),
rowset_ptrs(std::move(rowset_ptrs)),
delete_bitmap(std::move(db)) {}
int64_t max_version;
int64_t txn_id;
- const RowsetIdUnorderedSet& rowset_ids;
+ std::shared_ptr<RowsetIdUnorderedSet> rowset_ids;
std::vector<RowsetSharedPtr> rowset_ptrs;
std::shared_ptr<DeleteBitmap> delete_bitmap;
};
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index e230a25c874..a71eba21acb 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -326,7 +326,8 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
std::vector<RowsetSharedPtr> specified_rowsets;
{
std::shared_lock meta_rlock(_context.tablet->get_header_lock());
- specified_rowsets =
_context.tablet->get_rowset_by_ids(&_context.mow_context->rowset_ids);
+ specified_rowsets =
+
_context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get());
}
OlapStopWatch watch;
RETURN_IF_ERROR(BaseTablet::calc_delete_bitmap(
@@ -336,7 +337,7 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
segments.begin(), segments.end(), 0,
[](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum
+= s->num_rows(); });
LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " <<
_context.tablet->tablet_id()
- << ", rowset_ids: " << _context.mow_context->rowset_ids.size()
+ << ", rowset_ids: " << _context.mow_context->rowset_ids->size()
<< ", cur max_version: " << _context.mow_context->max_version
<< ", transaction_id: " << _context.mow_context->txn_id << ",
delete_bitmap_count: "
<< _context.mow_context->delete_bitmap->get_delete_bitmap_count()
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index be6c1dc45b8..8fd68b171d0 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -687,7 +687,7 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
if (config::enable_merge_on_write_correctness_check) {
_tablet->add_sentinel_mark_to_delete_bitmap(_mow_context->delete_bitmap.get(),
- _mow_context->rowset_ids);
+ *_mow_context->rowset_ids);
}
// read and fill block
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 6339c7db4bf..7bdc2aa3c21 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -548,7 +548,7 @@ Status
VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da
if (config::enable_merge_on_write_correctness_check) {
_tablet->add_sentinel_mark_to_delete_bitmap(_mow_context->delete_bitmap.get(),
- _mow_context->rowset_ids);
+ *_mow_context->rowset_ids);
}
// read and fill block
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index dfe4da0d55f..0cd0df391be 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -135,11 +135,11 @@ Status
BaseRowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_cont
"Unable to do 'partial_update' when "
"the tablet is undergoing a 'schema changing process'");
}
- _rowset_ids.clear();
+ _rowset_ids->clear();
} else {
RETURN_IF_ERROR(
- tablet()->get_all_rs_id_unlocked(_max_version_in_flush_phase,
&_rowset_ids));
- rowset_ptrs = tablet()->get_rowset_by_ids(&_rowset_ids);
+ tablet()->get_all_rs_id_unlocked(_max_version_in_flush_phase,
_rowset_ids.get()));
+ rowset_ptrs = tablet()->get_rowset_by_ids(_rowset_ids.get());
}
_delete_bitmap = std::make_shared<DeleteBitmap>(tablet()->tablet_id());
mow_context = std::make_shared<MowContext>(_max_version_in_flush_phase,
_req.txn_id,
@@ -289,7 +289,7 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() {
"partial update calc delete bitmap summary before commit:
tablet({}), txn_id({}), "
"rowset_ids({}), cur max_version({}), bitmap num({}),
bitmap_cardinality({}), num "
"rows updated({}), num rows new added({}), num rows
deleted({}), total rows({})",
- tablet()->tablet_id(), _req.txn_id, _rowset_ids.size(),
+ tablet()->tablet_id(), _req.txn_id, _rowset_ids->size(),
rowset_writer()->context().mow_context->max_version,
_delete_bitmap->get_delete_bitmap_count(),
_delete_bitmap->cardinality(),
rowset_writer()->num_rows_updated(),
rowset_writer()->num_rows_new_added(),
@@ -299,7 +299,7 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() {
LOG(INFO) << "submit calc delete bitmap task to executor, tablet_id: " <<
tablet()->tablet_id()
<< ", txn_id: " << _req.txn_id;
- return BaseTablet::commit_phase_update_delete_bitmap(_tablet, _rowset,
_rowset_ids,
+ return BaseTablet::commit_phase_update_delete_bitmap(_tablet, _rowset,
*_rowset_ids,
_delete_bitmap,
segments, _req.txn_id,
_calc_delete_bitmap_token.get(), nullptr);
}
@@ -319,7 +319,7 @@ Status RowsetBuilder::commit_txn() {
config::enable_merge_on_write_correctness_check && _rowset->num_rows()
!= 0 &&
tablet()->tablet_state() != TABLET_NOTREADY) {
auto st = tablet()->check_delete_bitmap_correctness(
- _delete_bitmap, _rowset->end_version() - 1, _req.txn_id,
_rowset_ids);
+ _delete_bitmap, _rowset->end_version() - 1, _req.txn_id,
*_rowset_ids);
if (!st.ok()) {
LOG(WARNING) << fmt::format(
"[tablet_id:{}][txn_id:{}][load_id:{}][partition_id:{}] "
@@ -364,7 +364,7 @@ Status RowsetBuilder::commit_txn() {
if (_tablet->enable_unique_key_merge_on_write()) {
_engine.txn_manager()->set_txn_related_delete_bitmap(
_req.partition_id, _req.txn_id, tablet()->tablet_id(),
tablet()->tablet_uid(), true,
- _delete_bitmap, _rowset_ids, _partial_update_info);
+ _delete_bitmap, *_rowset_ids, _partial_update_info);
}
_is_committed = true;
diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset_builder.h
index d87e2a9efa4..8da8fe8ad60 100644
--- a/be/src/olap/rowset_builder.h
+++ b/be/src/olap/rowset_builder.h
@@ -104,7 +104,7 @@ protected:
DeleteBitmapPtr _delete_bitmap;
std::unique_ptr<CalcDeleteBitmapToken> _calc_delete_bitmap_token;
// current rowset_ids, used to do diff in publish_version
- RowsetIdUnorderedSet _rowset_ids;
+ std::shared_ptr<RowsetIdUnorderedSet> _rowset_ids
{std::make_shared<RowsetIdUnorderedSet>()};
int64_t _max_version_in_flush_phase {-1};
std::shared_ptr<PartialUpdateInfo> _partial_update_info;
diff --git a/be/test/olap/segcompaction_mow_test.cpp
b/be/test/olap/segcompaction_mow_test.cpp
index efe40dcb859..18a0f241a2a 100644
--- a/be/test/olap/segcompaction_mow_test.cpp
+++ b/be/test/olap/segcompaction_mow_test.cpp
@@ -314,7 +314,7 @@ TEST_P(SegCompactionMoWTest, SegCompactionThenRead) {
RowsetWriterContext writer_context;
int raw_rsid = rand();
create_rowset_writer_context(raw_rsid, tablet_schema, &writer_context);
- RowsetIdUnorderedSet rsids;
+ std::shared_ptr<RowsetIdUnorderedSet> rsids
{std::make_shared<RowsetIdUnorderedSet>()};
std::vector<RowsetSharedPtr> rowset_ptrs;
writer_context.mow_context =
std::make_shared<MowContext>(1, 1, rsids, rowset_ptrs,
delete_bitmap);
@@ -417,7 +417,7 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
{ // write `num_segments * rows_per_segment` rows to rowset
RowsetWriterContext writer_context;
create_rowset_writer_context(20048, tablet_schema, &writer_context);
- RowsetIdUnorderedSet rsids;
+ std::shared_ptr<RowsetIdUnorderedSet> rsids
{std::make_shared<RowsetIdUnorderedSet>()};
std::vector<RowsetSharedPtr> rowset_ptrs;
writer_context.mow_context =
std::make_shared<MowContext>(1, 1, rsids, rowset_ptrs,
delete_bitmap);
@@ -648,7 +648,7 @@ TEST_F(SegCompactionMoWTest,
SegCompactionInterleaveWithBig_OoOoO) {
{ // write `num_segments * rows_per_segment` rows to rowset
RowsetWriterContext writer_context;
create_rowset_writer_context(20049, tablet_schema, &writer_context);
- RowsetIdUnorderedSet rsids;
+ std::shared_ptr<RowsetIdUnorderedSet> rsids
{std::make_shared<RowsetIdUnorderedSet>()};
std::vector<RowsetSharedPtr> rowset_ptrs;
writer_context.mow_context =
std::make_shared<MowContext>(1, 1, rsids, rowset_ptrs,
delete_bitmap);
@@ -838,7 +838,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionNotTrigger) {
{ // write `num_segments * rows_per_segment` rows to rowset
RowsetWriterContext writer_context;
create_rowset_writer_context(20050, tablet_schema, &writer_context);
- RowsetIdUnorderedSet rsids;
+ std::shared_ptr<RowsetIdUnorderedSet> rsids
{std::make_shared<RowsetIdUnorderedSet>()};
std::vector<RowsetSharedPtr> rowset_ptrs;
writer_context.mow_context =
std::make_shared<MowContext>(1, 1, rsids, rowset_ptrs,
delete_bitmap);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
