This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 58d0558203d [fix](mow) remove agg cache version for unused rowsets
(#49511)
58d0558203d is described below
commit 58d0558203dfe607afefe4330f92e80b78cdce20
Author: meiyi <me...@selectdb.com>
AuthorDate: Thu Mar 27 16:23:10 2025 +0800
[fix](mow) remove agg cache version for unused rowsets (#49511)
remove agg cache version for unused rowsets
---
be/src/olap/storage_engine.cpp | 6 +++++
be/src/olap/tablet_meta.cpp | 51 ++++++++++++++++++++++++++++++++++--------
be/src/olap/tablet_meta.h | 2 ++
3 files changed, 50 insertions(+), 9 deletions(-)
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 3ba65813492..84a654e366d 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -949,6 +949,11 @@ void StorageEngine::_clean_unused_rowset_metas() {
for (auto& rowset_meta : invalid_rowset_metas) {
static_cast<void>(RowsetMetaManager::remove(
data_dir->get_meta(), rowset_meta->tablet_uid(),
rowset_meta->rowset_id()));
+ TabletSharedPtr tablet =
_tablet_manager->get_tablet(rowset_meta->tablet_id());
+ if (tablet &&
tablet->tablet_meta()->enable_unique_key_merge_on_write()) {
+
tablet->tablet_meta()->delete_bitmap().remove_rowset_cache_version(
+ rowset_meta->rowset_id());
+ }
}
LOG(INFO) << "remove " << invalid_rowset_metas.size()
<< " invalid rowset meta from dir: " << data_dir->path();
@@ -1223,6 +1228,7 @@ void StorageEngine::start_delete_unused_rowset() {
tablet && tablet->enable_unique_key_merge_on_write()) {
tablet->tablet_meta()->delete_bitmap().remove({rs->rowset_id(), 0,
0},
{rs->rowset_id(),
UINT32_MAX, 0});
+
tablet->tablet_meta()->delete_bitmap().remove_rowset_cache_version(rs->rowset_id());
}
Status status = rs->remove();
unused_rowsets_counter << -1;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 7c45f3fe388..371282cd751 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1007,17 +1007,38 @@ void TabletMeta::clear_rowsets() {
}
void TabletMeta::_check_mow_rowset_cache_version_size(size_t
rowset_cache_version_size) {
- if (_enable_unique_key_merge_on_write &&
+ if (_enable_unique_key_merge_on_write && config::enable_mow_verbose_log &&
rowset_cache_version_size > _rs_metas.size() + _stale_rs_metas.size())
{
- std::string err_msg = fmt::format(
- ". tablet: {}, rowset_cache_version size: {}, "
- "_rs_metas size: {}, _stale_rs_metas size: {}",
- _tablet_id, rowset_cache_version_size, _rs_metas.size(),
_stale_rs_metas.size());
- if (config::enable_mow_get_agg_correctness_check_core) {
- CHECK(false) << err_msg;
- } else {
- DCHECK(false) << err_msg;
+ std::stringstream ss;
+ auto rowset_ids = _delete_bitmap->get_rowset_cache_version();
+ for (const auto& rowset_id : rowset_ids) {
+ bool found = false;
+ for (auto& rs_meta : _rs_metas) {
+ if (rs_meta->rowset_id() == rowset_id) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ continue;
+ }
+ for (auto& rs_meta : _stale_rs_metas) {
+ if (rs_meta->rowset_id() == rowset_id) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ ss << rowset_id.to_string() << ", ";
+ }
}
+ // size(rowset_cache_version) <= size(_rs_metas) +
size(_stale_rs_metas) + size(_unused_rs)
+ std::string msg = fmt::format(
+ "tablet: {}, rowset_cache_version size: {}, "
+ "_rs_metas size: {}, _stale_rs_metas size: {}, delta: {}.
rowset only in cache: {}",
+ _tablet_id, rowset_cache_version_size, _rs_metas.size(),
_stale_rs_metas.size(),
+ rowset_cache_version_size - _rs_metas.size() -
_stale_rs_metas.size(), ss.str());
+ LOG(INFO) << msg;
}
}
@@ -1337,12 +1358,24 @@ bool
DeleteBitmap::has_calculated_for_multi_segments(const RowsetId& rowset_id)
size_t DeleteBitmap::remove_rowset_cache_version(const RowsetId& rowset_id) {
std::lock_guard l(_rowset_cache_version_lock);
_rowset_cache_version.erase(rowset_id);
+ VLOG_DEBUG << "remove agg cache version for tablet=" << _tablet_id
+ << ", rowset=" << rowset_id.to_string();
return _rowset_cache_version.size();
}
void DeleteBitmap::clear_rowset_cache_version() {
std::lock_guard l(_rowset_cache_version_lock);
_rowset_cache_version.clear();
+ VLOG_DEBUG << "clear agg cache version for tablet=" << _tablet_id;
+}
+
+std::set<RowsetId> DeleteBitmap::get_rowset_cache_version() {
+ std::set<RowsetId> set;
+ std::shared_lock l(_rowset_cache_version_lock);
+ for (auto& [k, _] : _rowset_cache_version) {
+ set.insert(k);
+ }
+ return set;
}
DeleteBitmap::Version DeleteBitmap::_get_rowset_cache_version(const BitmapKey&
bmk) const {
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 47294e7c008..a235a103258 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -564,6 +564,8 @@ public:
void clear_rowset_cache_version();
+ std::set<RowsetId> get_rowset_cache_version();
+
class AggCachePolicy : public LRUCachePolicy {
public:
AggCachePolicy(size_t capacity)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
