This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 98ba8ae0eb4 branch-4.2: [fix](mow) Skip delete bitmap cache prefill
for non-running tablets #68436 (#69497)
98ba8ae0eb4 is described below
commit 98ba8ae0eb4924d9a8d33912ca002421f5da6b0d
Author: bobhan1 <[email protected]>
AuthorDate: Sat Oct 24 10:27:18 2026 +0800
branch-4.2: [fix](mow) Skip delete bitmap cache prefill for non-running
tablets #68436 (#69497)
### What problem does this PR solve?
Backport #68436 to `branch-4.2` (upstream commit
`e4d328e23b888ea9698f7e58c99dcf460580e122`).
During cloud schema change, compaction can prefill the aggregate
delete-bitmap cache before a `TABLET_NOTREADY` tablet has its complete
delete bitmap. Those entries can survive the transition to
`TABLET_RUNNING` and expose overwritten rows. Require `TABLET_RUNNING`
under the existing tablet header read lock before either post-compaction
prefill mode runs.
The production patch is identical to upstream. Adapt the parameterized
test to 4.2's contiguous segment IDs and `DeleteBitmap::BitmapKey` API,
retaining all four prefill switch combinations, output and non-output
rowsets, and the unchanged read-version check across the state
transition.
### Release note
Fix duplicate rows returned by Unique Key Merge-on-Write queries when
schema change overlaps compaction.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test
- [ ] No need to test or manual test.
Validation:
```bash
./run-be-ut.sh --run
--filter='PrefillModes/CloudTabletDeleteBitmapPrefillTest.*:CloudTabletDeleteBitmapTest.*:TabletMetaTest.*:CloudCompactionTest.*'
-j100
```
ASAN: **52 tests from 4 test suites passed**, including all four new
prefill parameter combinations, `CloudTabletDeleteBitmapTest`,
`TabletMetaTest`, and `CloudCompactionTest`.
Changed C++ code passes clang-format 16; `git diff --check
upstream/branch-4.2...HEAD` passes. Docker regression was not run.
- Behavior changed:
- [ ] No.
- [x] Yes. Skip post-compaction aggregate delete-bitmap cache prefill
until the tablet is RUNNING.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/src/storage/tablet/base_tablet.cpp | 5 +++
be/test/cloud/cloud_tablet_test.cpp | 75 +++++++++++++++++++++++++++++++++++
2 files changed, 80 insertions(+)
diff --git a/be/src/storage/tablet/base_tablet.cpp
b/be/src/storage/tablet/base_tablet.cpp
index 9c985b38ffd..fad76e86986 100644
--- a/be/src/storage/tablet/base_tablet.cpp
+++ b/be/src/storage/tablet/base_tablet.cpp
@@ -2443,6 +2443,11 @@ void
BaseTablet::prefill_dbm_agg_cache_after_compaction(const RowsetSharedPtr& o
int64_t cur_max_version {-1};
{
std::shared_lock rlock(get_header_lock());
+ // Schema change may still be rebuilding the delete bitmap of a
NOTREADY tablet.
+ // Prefilling now can cache incomplete bitmaps that remain stale
after it becomes RUNNING.
+ if (tablet_state() != TABLET_RUNNING) {
+ return;
+ }
cur_max_version = max_version_unlocked();
}
if (config::enable_prefill_all_dbm_agg_cache_after_compaction) {
diff --git a/be/test/cloud/cloud_tablet_test.cpp
b/be/test/cloud/cloud_tablet_test.cpp
index 913d98e794b..9eccb83fedb 100644
--- a/be/test/cloud/cloud_tablet_test.cpp
+++ b/be/test/cloud/cloud_tablet_test.cpp
@@ -38,7 +38,9 @@
#include "storage/rowset/rowset.h"
#include "storage/rowset/rowset_factory.h"
#include "storage/rowset/rowset_meta.h"
+#include "storage/rowset/unique_rowset_id_generator.h"
#include "storage/tablet/tablet_meta.h"
+#include "util/defer_op.h"
#include "util/uid_util.h"
namespace doris {
@@ -337,6 +339,79 @@ TEST_F(CloudTabletDeleteBitmapTest,
AggDeleteBitmapForCompactionReturnsPreRowset
EXPECT_EQ(rowset_versions_without_stats, pre_rowset_to_versions);
}
+class CloudTabletDeleteBitmapPrefillTest
+ : public CloudTabletWarmUpStateTest,
+ public testing::WithParamInterface<std::tuple<bool, bool>> {
+public:
+ void SetUp() override {
+ CloudTabletWarmUpStateTest::SetUp();
+ // The aggregate cache is global, so rowset IDs must be unique across
test instances.
+ _engine._rowset_id_generator =
+ std::make_unique<UniqueRowsetIdGenerator>(UniqueId::gen_uid());
+ }
+};
+
+TEST_P(CloudTabletDeleteBitmapPrefillTest, WaitForSchemaChangeDeleteBitmap) {
+ const auto old_prefill_output =
config::enable_prefill_output_dbm_agg_cache_after_compaction;
+ const auto old_prefill_all =
config::enable_prefill_all_dbm_agg_cache_after_compaction;
+ Defer restore_config {[&] {
+ config::enable_prefill_output_dbm_agg_cache_after_compaction =
old_prefill_output;
+ config::enable_prefill_all_dbm_agg_cache_after_compaction =
old_prefill_all;
+ }};
+ const auto [prefill_output, prefill_all] = GetParam();
+ config::enable_prefill_output_dbm_agg_cache_after_compaction =
prefill_output;
+ config::enable_prefill_all_dbm_agg_cache_after_compaction = prefill_all;
+
+ _tablet->tablet_meta()->set_enable_unique_key_merge_on_write(true);
+ _tablet->tablet_meta()->mutable_tablet_schema()->_keys_type = UNIQUE_KEYS;
+ auto output_rowset = create_rowset(Version(2, 3), 2);
+ auto other_rowset = create_rowset(Version(4, 4), 2);
+ ASSERT_NE(output_rowset, nullptr);
+ ASSERT_NE(other_rowset, nullptr);
+ const std::vector<RowsetSharedPtr> rowsets {output_rowset, other_rowset};
+ {
+ std::unique_lock lock(_tablet->get_header_lock());
+ ASSERT_TRUE(_tablet->set_tablet_state(TABLET_NOTREADY).ok());
+ _tablet->add_rowsets(rowsets, false, lock, false);
+ }
+
+ // Compaction finishes before schema change installs the complete delete
bitmap.
+ _tablet->prefill_dbm_agg_cache_after_compaction(output_rowset);
+ auto cache_before =
DeleteBitmapAggCache::instance()->snapshot(_tablet->tablet_id());
+ DeleteBitmap complete_delete_bitmap(_tablet->tablet_id());
+ for (const auto& rowset : rowsets) {
+ for (uint32_t segment_id = 0; segment_id < rowset->num_segments();
++segment_id) {
+ DeleteBitmap::BitmapKey key {rowset->rowset_id(), segment_id, 4};
+ EXPECT_EQ(cache_before.get(key), nullptr);
+ complete_delete_bitmap.add(key, 1);
+ }
+ }
+ {
+ std::unique_lock lock(_tablet->get_header_lock());
+ _tablet->tablet_meta()->delete_bitmap() = complete_delete_bitmap;
+ ASSERT_TRUE(_tablet->set_tablet_state(TABLET_RUNNING).ok());
+ }
+
+ // Prefill must still work for RUNNING tablets, honoring both
configuration switches.
+ _tablet->prefill_dbm_agg_cache_after_compaction(output_rowset);
+ auto cache_after =
DeleteBitmapAggCache::instance()->snapshot(_tablet->tablet_id());
+ for (const auto& rowset : rowsets) {
+ const bool should_prefill = prefill_all || (prefill_output && rowset
== output_rowset);
+ for (uint32_t segment_id = 0; segment_id < rowset->num_segments();
++segment_id) {
+ DeleteBitmap::BitmapKey key {rowset->rowset_id(), segment_id, 4};
+ EXPECT_EQ(cache_after.get(key) != nullptr, should_prefill);
+ // The read version and rowset IDs are unchanged after schema
change. A premature
+ // prefill would return the old empty bitmap and expose
overwritten rows here.
+ auto bitmap = _tablet->tablet_meta()->delete_bitmap().get_agg(key);
+ EXPECT_EQ(bitmap->cardinality(), 1);
+ EXPECT_TRUE(bitmap->contains(1));
+ }
+ }
+}
+
+INSTANTIATE_TEST_SUITE_P(PrefillModes, CloudTabletDeleteBitmapPrefillTest,
+ testing::Combine(testing::Bool(), testing::Bool()));
+
// Test get_rowset_warmup_state for non-existent rowset
TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) {
auto rowset = create_rowset(Version(1, 1));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
