This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0ab43c51e8 [Feature](unique-key-merge-on-write) some fix on delete
bitmap usage (#11623)
0ab43c51e8 is described below
commit 0ab43c51e8fd1bbc35a8b183521e0efbe223ad3e
Author: zhannngchen <48427519+zhannngc...@users.noreply.github.com>
AuthorDate: Fri Aug 12 11:54:31 2022 +0800
[Feature](unique-key-merge-on-write) some fix on delete bitmap usage
(#11623)
---
be/src/olap/delta_writer.cpp | 17 ++++++++++-------
be/src/olap/merger.cpp | 6 +++++-
be/src/olap/rowset/beta_rowset_reader.cpp | 2 ++
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 3 +++
be/src/olap/tablet.cpp | 11 +++++++----
be/src/olap/tablet_meta.cpp | 2 +-
be/src/vec/exec/volap_scanner.cpp | 4 +++-
7 files changed, 31 insertions(+), 14 deletions(-)
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index 1d1ec6c01d..89ce579b5a 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -101,6 +101,13 @@ Status DeltaWriter::init() {
<< ", schema_hash=" << _req.schema_hash;
return Status::OLAPInternalError(OLAP_ERR_TABLE_NOT_FOUND);
}
+
+ // get rowset ids snapshot
+ if (_tablet->enable_unique_key_merge_on_write()) {
+ std::lock_guard<std::shared_mutex> lck(_tablet->get_header_lock());
+ _rowset_ids = _tablet->all_rs_id();
+ }
+
_mem_tracker = std::make_shared<MemTrackerLimiter>(
-1, fmt::format("DeltaWriter:tabletId={}", _tablet->tablet_id()),
_parent_tracker);
SCOPED_ATTACH_TASK(_mem_tracker, ThreadContext::TaskType::LOAD);
@@ -138,13 +145,6 @@ Status DeltaWriter::init() {
RETURN_NOT_OK(_storage_engine->memtable_flush_executor()->create_flush_token(
&_flush_token, _rowset_writer->type(), _req.is_high_priority));
- // create delete bitmap and get rowset ids snapshot
- if (_tablet->enable_unique_key_merge_on_write()) {
- _delete_bitmap = std::make_shared<DeleteBitmap>(-1);
- std::lock_guard<std::shared_mutex> lck(_tablet->get_header_lock());
- _rowset_ids = _tablet->all_rs_id();
- }
-
_is_init = true;
return Status::OK();
}
@@ -289,6 +289,9 @@ Status DeltaWriter::wait_flush() {
}
void DeltaWriter::_reset_mem_table() {
+ if (_tablet->enable_unique_key_merge_on_write()) {
+ _delete_bitmap.reset(new DeleteBitmap(-1));
+ }
_mem_table.reset(new MemTable(_tablet, _schema.get(),
_tablet_schema.get(), _req.slots,
_req.tuple_desc, _rowset_writer.get(),
_delete_bitmap,
_rowset_ids, _is_vec));
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 91c87bb081..fa7b99a973 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -115,7 +115,11 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
std::inserter(reader_params.delete_predicates,
reader_params.delete_predicates.begin()));
}
- reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
+
+ if (tablet->enable_unique_key_merge_on_write()) {
+ reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
+ }
+
if (stats_output && stats_output->rowid_conversion) {
reader_params.record_rowids = true;
}
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 3c848a43b6..e1276d3fa8 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -93,6 +93,8 @@ Status BetaRowsetReader::init(RowsetReaderContext*
read_context) {
auto d = read_context->delete_bitmap->get_agg(
{rowset_id, seg_id, read_context->version.second});
if (d->isEmpty()) continue; // Empty delete bitmap for the segment
+ VLOG_TRACE << "Get the delete bitmap for rowset: " <<
rowset_id.to_string()
+ << ", segment id:" << seg_id << ", size:" <<
d->cardinality();
read_options.delete_bitmap.emplace(seg_id, std::move(d));
}
}
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 3b91bbc024..cc728bdbe7 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -197,6 +197,9 @@ Status SegmentIterator::_init(bool is_vec) {
size_t pre_size = _row_bitmap.cardinality();
_row_bitmap -= *(_opts.delete_bitmap[segment_id()]);
_opts.stats->rows_del_by_bitmap += (pre_size -
_row_bitmap.cardinality());
+ VLOG_DEBUG << "read on segment: " << segment_id() << ", delete bitmap
cardinality: "
+ << _opts.delete_bitmap[segment_id()]->cardinality() << ", "
+ << _opts.stats->rows_del_by_bitmap << " rows deleted by
bitmap";
}
if (_opts.read_orderby_key_reverse) {
_range_iter.reset(new BackwardBitmapRangeIterator(_row_bitmap));
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 86ff3dd2d9..bf209a52f0 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1994,8 +1994,9 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id,
}
}
LOG(INFO) << "construct delete bitmap tablet: " << tablet_id() << "
rowset: " << rowset_id
- << " dummy_version: " << dummy_version << " cost: " <<
watch.get_elapse_time_us()
- << "(us)";
+ << " dummy_version: " << dummy_version
+ << "bitmap num: " << delete_bitmap->delete_bitmap.size()
+ << " cost: " << watch.get_elapse_time_us() << "(us)";
return Status::OK();
}
@@ -2043,8 +2044,10 @@ Status Tablet::update_delete_bitmap(const
RowsetSharedPtr& rowset, DeleteBitmapP
std::lock_guard<std::shared_mutex> meta_wrlock(_meta_lock);
cur_rowset_ids = all_rs_id();
_rowset_ids_difference(cur_rowset_ids, pre_rowset_ids, &rowset_ids_to_add,
&rowset_ids_to_del);
- LOG(INFO) << "rowset_ids_to_add: " << rowset_ids_to_add.size()
- << ", rowset_ids_to_del: " << rowset_ids_to_del.size();
+ if (!rowset_ids_to_add.empty() || !rowset_ids_to_del.empty()) {
+ LOG(INFO) << "rowset_ids_to_add: " << rowset_ids_to_add.size()
+ << ", rowset_ids_to_del: " << rowset_ids_to_del.size();
+ }
for (const auto& to_del : rowset_ids_to_del) {
delete_bitmap->remove({to_del, 0, 0}, {to_del, UINT32_MAX, INT64_MAX});
}
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index c1bc473bbd..b2dd33b281 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -783,7 +783,7 @@ void TabletMeta::update_delete_bitmap(const
std::vector<RowsetSharedPtr>& input_
for (auto index = iter->second.begin(); index !=
iter->second.end(); ++index) {
src.row_id = *index;
if (rowid_conversion.get(src, &dst) != 0) {
- LOG(WARNING) << "Can't find rowid, may be deleted by
the delete_handler.";
+ VLOG_CRITICAL << "Can't find rowid, may be deleted by
the delete_handler.";
continue;
}
output_rowset_delete_bitmap.add({dst.rowset_id,
dst.segment_id, cur_version},
diff --git a/be/src/vec/exec/volap_scanner.cpp
b/be/src/vec/exec/volap_scanner.cpp
index 2a454c3f88..beb04ab0b8 100644
--- a/be/src/vec/exec/volap_scanner.cpp
+++ b/be/src/vec/exec/volap_scanner.cpp
@@ -242,7 +242,9 @@ Status VOlapScanner::_init_tablet_reader_params(
_tablet_reader_params.use_page_cache = true;
}
- _tablet_reader_params.delete_bitmap =
&_tablet->tablet_meta()->delete_bitmap();
+ if (_tablet->enable_unique_key_merge_on_write()) {
+ _tablet_reader_params.delete_bitmap =
&_tablet->tablet_meta()->delete_bitmap();
+ }
if (_parent->_olap_scan_node.__isset.sort_info &&
_parent->_olap_scan_node.sort_info.is_asc_order.size() > 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
