This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d6a514e0dd0 branch-3.1: [Fix](warmup) one-time and periodic warm up
job should be triggered regardless whether the rowset has been warmed up before
#56737 (#56864)
d6a514e0dd0 is described below
commit d6a514e0dd0450d80aab7dfbbf20b43ab1f41853
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Oct 13 12:13:40 2025 +0800
branch-3.1: [Fix](warmup) one-time and periodic warm up job should be
triggered regardless whether the rowset has been warmed up before #56737
(#56864)
Cherry-picked from #56737
Co-authored-by: bobhan1 <[email protected]>
---
be/src/cloud/cloud_internal_service.cpp | 16 +-
be/src/cloud/cloud_tablet.cpp | 129 +++++---
be/src/cloud/cloud_tablet.h | 34 ++-
be/src/cloud/cloud_warm_up_manager.cpp | 29 +-
be/src/cloud/cloud_warm_up_manager.h | 7 -
be/test/cloud/cloud_tablet_test.cpp | 513 ++++++++++++++++++++++++++------
6 files changed, 557 insertions(+), 171 deletions(-)
diff --git a/be/src/cloud/cloud_internal_service.cpp
b/be/src/cloud/cloud_internal_service.cpp
index 2e058571ba0..905612d236a 100644
--- a/be/src/cloud/cloud_internal_service.cpp
+++ b/be/src/cloud/cloud_internal_service.cpp
@@ -20,6 +20,7 @@
#include <bthread/countdown_event.h>
#include "cloud/cloud_storage_engine.h"
+#include "cloud/cloud_tablet.h"
#include "cloud/cloud_tablet_mgr.h"
#include "cloud/cloud_warm_up_manager.h"
#include "cloud/config.h"
@@ -225,7 +226,7 @@ void
CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c
expiration_time = 0;
}
- if (!tablet->add_rowset_warmup_state(rs_meta,
WarmUpState::TRIGGERED_BY_JOB)) {
+ if (!tablet->add_rowset_warmup_state(rs_meta,
WarmUpTriggerSource::EVENT_DRIVEN)) {
LOG(INFO) << "found duplicate warmup task for rowset " <<
rowset_id.to_string()
<< ", skip it";
continue;
@@ -276,8 +277,9 @@ void
CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c
LOG(WARNING) << "download segment failed, tablet_id: " <<
tablet_id
<< " rowset_id: " << rowset_id.to_string() <<
", error: " << st;
}
- if (tablet->complete_rowset_segment_warmup(rowset_id, st, 1,
0) ==
- WarmUpState::DONE) {
+ if
(tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN,
+ rowset_id, st, 1, 0)
+ .trigger_source ==
WarmUpTriggerSource::EVENT_DRIVEN) {
VLOG_DEBUG << "warmup rowset " << version.to_string() <<
"("
<< rowset_id.to_string() << ") completed";
}
@@ -350,8 +352,9 @@ void
CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c
<< "download inverted index failed, tablet_id:
" << tablet_id
<< " rowset_id: " << rowset_id.to_string() <<
", error: " << st;
}
- if (tablet->complete_rowset_segment_warmup(rowset_id, st,
0, 1) ==
- WarmUpState::DONE) {
+ if
(tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN,
+ rowset_id, st,
0, 1)
+ .trigger_source ==
WarmUpTriggerSource::EVENT_DRIVEN) {
VLOG_DEBUG << "warmup rowset " << version.to_string()
<< "("
<< rowset_id.to_string() << ") completed";
}
@@ -371,7 +374,8 @@ void
CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c
};
g_file_cache_event_driven_warm_up_submitted_index_num << 1;
g_file_cache_event_driven_warm_up_submitted_index_size <<
idx_size;
- tablet->update_rowset_warmup_state_inverted_idx_num(rowset_id,
1);
+ tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset_id, 1);
if (wait) {
wait->add_count();
}
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 4913f657f31..1209b010a9f 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -122,6 +122,8 @@ bvar::Adder<uint64_t>
g_file_cache_warm_up_rowset_triggered_by_job_num(
"file_cache_warm_up_rowset_triggered_by_job_num");
bvar::Adder<uint64_t> g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num(
"file_cache_warm_up_rowset_triggered_by_sync_rowset_num");
+bvar::Adder<uint64_t>
g_file_cache_warm_up_rowset_triggered_by_event_driven_num(
+ "file_cache_warm_up_rowset_triggered_by_event_driven_num");
bvar::LatencyRecorder g_file_cache_warm_up_rowset_all_segments_latency(
"file_cache_warm_up_rowset_all_segments_latency");
@@ -445,8 +447,8 @@ void CloudTablet::add_rowsets(std::vector<RowsetSharedPtr>
to_add, bool version_
if (!warm_up_state_updated) {
VLOG_DEBUG << "warm up rowset " << rs->version() <<
"(" << rs->rowset_id()
<< ") triggerd by sync rowset";
- if (!add_rowset_warmup_state_unlocked(
- *(rs->rowset_meta()),
WarmUpState::TRIGGERED_BY_SYNC_ROWSET)) {
+ if
(!add_rowset_warmup_state_unlocked(*(rs->rowset_meta()),
+
WarmUpTriggerSource::SYNC_ROWSET)) {
LOG(INFO) << "found duplicate warmup task for
rowset "
<< rs->rowset_id() << ", skip it";
break;
@@ -479,7 +481,7 @@ void CloudTablet::add_rowsets(std::vector<RowsetSharedPtr>
to_add, bool version_
std::chrono::seconds(sleep_time));
}
});
-
self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st, 1, 0);
+
self->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET,
rowset_meta->rowset_id(), st, 1, 0);
if (!st) {
LOG_WARNING("add rowset warm up error
").error(st);
}
@@ -511,13 +513,13 @@ void
CloudTablet::add_rowsets(std::vector<RowsetSharedPtr> to_add, bool version_
std::chrono::seconds(sleep_time));
// clang-format off
});
-
self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st, 0, 1);
+
self->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET,
rowset_meta->rowset_id(), st, 0, 1);
if (!st) {
LOG_WARNING("add rowset warm up error
").error(st);
}
}},
};
-
self->update_rowset_warmup_state_inverted_idx_num_unlocked(rowset_meta->rowset_id(),
1);
+
self->update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource::SYNC_ROWSET,
rowset_meta->rowset_id(), 1);
_engine.file_cache_block_downloader().submit_download_task(std::move(meta));
g_file_cache_cloud_tablet_submitted_index_num << 1;
g_file_cache_cloud_tablet_submitted_index_size <<
idx_size;
@@ -1504,51 +1506,103 @@ Status CloudTablet::check_delete_bitmap_cache(int64_t
txn_id,
WarmUpState CloudTablet::get_rowset_warmup_state(RowsetId rowset_id) {
std::shared_lock rlock(_meta_lock);
if (!_rowset_warm_up_states.contains(rowset_id)) {
- return WarmUpState::NONE;
+ return {.trigger_source = WarmUpTriggerSource::NONE, .progress =
WarmUpProgress::NONE};
}
- return _rowset_warm_up_states[rowset_id].state;
+ auto& warmup_info = _rowset_warm_up_states[rowset_id];
+ warmup_info.update_state();
+ return warmup_info.state;
}
-bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset,
WarmUpState state,
+bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset,
WarmUpTriggerSource source,
std::chrono::steady_clock::time_point start_tp) {
std::lock_guard wlock(_meta_lock);
- return add_rowset_warmup_state_unlocked(rowset, state, start_tp);
+ return add_rowset_warmup_state_unlocked(rowset, source, start_tp);
}
-void CloudTablet::update_rowset_warmup_state_inverted_idx_num(RowsetId
rowset_id, int64_t delta) {
+bool
CloudTablet::update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource
source,
+ RowsetId
rowset_id, int64_t delta) {
std::lock_guard wlock(_meta_lock);
- update_rowset_warmup_state_inverted_idx_num_unlocked(rowset_id, delta);
+ return update_rowset_warmup_state_inverted_idx_num_unlocked(source,
rowset_id, delta);
}
-void
CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId
rowset_id,
+bool
CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource
source,
+
RowsetId rowset_id,
int64_t
delta) {
- if (!_rowset_warm_up_states.contains(rowset_id)) {
- return;
+ auto it = _rowset_warm_up_states.find(rowset_id);
+ if (it == _rowset_warm_up_states.end()) {
+ return false;
+ }
+ if (it->second.state.trigger_source != source) {
+ // Only the same trigger source can update the state
+ return false;
}
- _rowset_warm_up_states[rowset_id].num_inverted_idx += delta;
+ it->second.num_inverted_idx += delta;
+ return true;
}
-bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset,
WarmUpState state,
+bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset,
+ WarmUpTriggerSource source,
std::chrono::steady_clock::time_point start_tp) {
- if (_rowset_warm_up_states.contains(rowset.rowset_id())) {
- return false;
+ auto rowset_id = rowset.rowset_id();
+
+ // Check if rowset already has warmup state
+ if (_rowset_warm_up_states.contains(rowset_id)) {
+ auto existing_state = _rowset_warm_up_states[rowset_id].state;
+
+ // For job-triggered warmup (one-time and periodic warmup), allow it
to proceed
+ // except when there's already another job-triggered warmup in progress
+ if (source == WarmUpTriggerSource::JOB) {
+ if (existing_state.trigger_source == WarmUpTriggerSource::JOB &&
+ existing_state.progress == WarmUpProgress::DOING) {
+ // Same job type already in progress, skip to avoid duplicate
warmup
+ return false;
+ }
+ } else {
+ // For non-job warmup (EVENT_DRIVEN, SYNC_ROWSET), skip if any
warmup exists
+ return false;
+ }
}
- if (state == WarmUpState::TRIGGERED_BY_JOB) {
+
+ if (source == WarmUpTriggerSource::JOB) {
g_file_cache_warm_up_rowset_triggered_by_job_num << 1;
- } else if (state == WarmUpState::TRIGGERED_BY_SYNC_ROWSET) {
+ } else if (source == WarmUpTriggerSource::SYNC_ROWSET) {
g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num << 1;
- }
- _rowset_warm_up_states[rowset.rowset_id()] = {
- .state = state, .num_segments = rowset.num_segments(), .start_tp =
start_tp};
+ } else if (source == WarmUpTriggerSource::EVENT_DRIVEN) {
+ g_file_cache_warm_up_rowset_triggered_by_event_driven_num << 1;
+ }
+ _rowset_warm_up_states[rowset_id] = {
+ .state = {.trigger_source = source,
+ .progress = (rowset.num_segments() == 0 ?
WarmUpProgress::DONE
+ :
WarmUpProgress::DOING)},
+ .num_segments = rowset.num_segments(),
+ .start_tp = start_tp};
return true;
}
-WarmUpState CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id,
Status status,
+void CloudTablet::RowsetWarmUpInfo::update_state() {
+ if (has_finished()) {
+ g_file_cache_warm_up_rowset_complete_num << 1;
+ auto cost = std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::steady_clock::now() - start_tp)
+ .count();
+ g_file_cache_warm_up_rowset_all_segments_latency << cost;
+ state.progress = WarmUpProgress::DONE;
+ }
+}
+
+WarmUpState CloudTablet::complete_rowset_segment_warmup(WarmUpTriggerSource
trigger_source,
+ RowsetId rowset_id,
Status status,
int64_t segment_num,
int64_t
inverted_idx_num) {
std::lock_guard wlock(_meta_lock);
- if (!_rowset_warm_up_states.contains(rowset_id)) {
- return WarmUpState::NONE;
+ auto it = _rowset_warm_up_states.find(rowset_id);
+ if (it == _rowset_warm_up_states.end()) {
+ return {.trigger_source = WarmUpTriggerSource::NONE, .progress =
WarmUpProgress::NONE};
+ }
+ auto& warmup_info = it->second;
+ if (warmup_info.state.trigger_source != trigger_source) {
+ // Only the same trigger source can update the state
+ return warmup_info.state;
}
VLOG_DEBUG << "complete rowset segment warmup for rowset " << rowset_id <<
", " << status;
if (segment_num > 0) {
@@ -1563,17 +1617,8 @@ WarmUpState
CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id, Stat
g_file_cache_warm_up_inverted_idx_failed_num << inverted_idx_num;
}
}
- _rowset_warm_up_states[rowset_id].done(segment_num, inverted_idx_num);
- if (_rowset_warm_up_states[rowset_id].has_finished()) {
- g_file_cache_warm_up_rowset_complete_num << 1;
- auto cost = std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::steady_clock::now() -
- _rowset_warm_up_states[rowset_id].start_tp)
- .count();
- g_file_cache_warm_up_rowset_all_segments_latency << cost;
- _rowset_warm_up_states[rowset_id].state = WarmUpState::DONE;
- }
- return _rowset_warm_up_states[rowset_id].state;
+ warmup_info.done(segment_num, inverted_idx_num);
+ return warmup_info.state;
}
bool CloudTablet::is_rowset_warmed_up(const RowsetId& rowset_id) const {
@@ -1581,13 +1626,15 @@ bool CloudTablet::is_rowset_warmed_up(const RowsetId&
rowset_id) const {
if (it == _rowset_warm_up_states.end()) {
return false;
}
- return it->second.state == WarmUpState::DONE;
+ return it->second.state.progress == WarmUpProgress::DONE;
}
void CloudTablet::add_warmed_up_rowset(const RowsetId& rowset_id) {
- _rowset_warm_up_states[rowset_id] = {.state = WarmUpState::DONE,
- .num_segments = 1,
- .start_tp =
std::chrono::steady_clock::now()};
+ _rowset_warm_up_states[rowset_id] = {
+ .state = {.trigger_source = WarmUpTriggerSource::SYNC_ROWSET,
+ .progress = WarmUpProgress::DONE},
+ .num_segments = 1,
+ .start_tp = std::chrono::steady_clock::now()};
}
#include "common/compile_check_end.h"
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index 109408bf4bb..6ecd2b3762c 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -26,7 +26,19 @@
namespace doris {
class CloudStorageEngine;
-enum class WarmUpState : int;
+
+enum class WarmUpTriggerSource : int { NONE, SYNC_ROWSET, EVENT_DRIVEN, JOB };
+
+enum class WarmUpProgress : int { NONE, DOING, DONE };
+
+struct WarmUpState {
+ WarmUpTriggerSource trigger_source {WarmUpTriggerSource::NONE};
+ WarmUpProgress progress {WarmUpProgress::NONE};
+
+ bool operator==(const WarmUpState& other) const {
+ return trigger_source == other.trigger_source && progress ==
other.progress;
+ }
+};
struct SyncRowsetStats {
int64_t get_remote_rowsets_num {0};
@@ -312,11 +324,14 @@ public:
// Add warmup state management
WarmUpState get_rowset_warmup_state(RowsetId rowset_id);
bool add_rowset_warmup_state(
- const RowsetMeta& rowset, WarmUpState state,
+ const RowsetMeta& rowset, WarmUpTriggerSource source,
std::chrono::steady_clock::time_point start_tp =
std::chrono::steady_clock::now());
- void update_rowset_warmup_state_inverted_idx_num(RowsetId rowset_id,
int64_t delta);
- void update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId
rowset_id, int64_t delta);
- WarmUpState complete_rowset_segment_warmup(RowsetId rowset_id, Status
status,
+ bool update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource
source, RowsetId rowset_id,
+ int64_t delta);
+ bool
update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source,
+ RowsetId
rowset_id, int64_t delta);
+ WarmUpState complete_rowset_segment_warmup(WarmUpTriggerSource
trigger_source,
+ RowsetId rowset_id, Status
status,
int64_t segment_num, int64_t
inverted_idx_num);
bool is_rowset_warmed_up(const RowsetId& rowset_id) const;
@@ -329,8 +344,8 @@ public:
auto tmp = fmt::format("{}{}", rs->rowset_id().to_string(),
rs->version().to_string());
if (_rowset_warm_up_states.contains(rs->rowset_id())) {
tmp += fmt::format(
- ", state={}, segments_warmed_up={}/{},
inverted_idx_warmed_up={}/{}",
- _rowset_warm_up_states.at(rs->rowset_id()).state,
+ ", progress={}, segments_warmed_up={}/{},
inverted_idx_warmed_up={}/{}",
+
_rowset_warm_up_states.at(rs->rowset_id()).state.progress,
_rowset_warm_up_states.at(rs->rowset_id()).num_segments_warmed_up,
_rowset_warm_up_states.at(rs->rowset_id()).num_segments,
_rowset_warm_up_states.at(rs->rowset_id()).num_inverted_idx_warmed_up,
@@ -349,7 +364,7 @@ private:
Status sync_if_not_running(SyncRowsetStats* stats = nullptr);
bool add_rowset_warmup_state_unlocked(
- const RowsetMeta& rowset, WarmUpState state,
+ const RowsetMeta& rowset, WarmUpTriggerSource source,
std::chrono::steady_clock::time_point start_tp =
std::chrono::steady_clock::now());
// used by capture_rs_reader_xxx functions
@@ -427,12 +442,15 @@ private:
void done(int64_t num_segments, int64_t num_inverted_idx) {
num_segments_warmed_up += num_segments;
num_inverted_idx_warmed_up += num_inverted_idx;
+ update_state();
}
bool has_finished() const {
return (num_segments_warmed_up >= num_segments) &&
(num_inverted_idx_warmed_up >= num_inverted_idx);
}
+
+ void update_state();
};
std::unordered_map<RowsetId, RowsetWarmUpInfo> _rowset_warm_up_states;
diff --git a/be/src/cloud/cloud_warm_up_manager.cpp
b/be/src/cloud/cloud_warm_up_manager.cpp
index c87de1f5fe4..b39b141f366 100644
--- a/be/src/cloud/cloud_warm_up_manager.cpp
+++ b/be/src/cloud/cloud_warm_up_manager.cpp
@@ -25,6 +25,7 @@
#include <tuple>
#include "bvar/bvar.h"
+#include "cloud/cloud_tablet.h"
#include "cloud/cloud_tablet_mgr.h"
#include "cloud/config.h"
#include "common/logging.h"
@@ -225,7 +226,7 @@ void CloudWarmUpManager::handle_jobs() {
if (expiration_time <= UnixSeconds()) {
expiration_time = 0;
}
- if (!tablet->add_rowset_warmup_state(*rs,
WarmUpState::TRIGGERED_BY_JOB)) {
+ if (!tablet->add_rowset_warmup_state(*rs,
WarmUpTriggerSource::JOB)) {
LOG(INFO) << "found duplicate warmup task for rowset " <<
rs->rowset_id()
<< ", skip it";
continue;
@@ -238,8 +239,10 @@ void CloudWarmUpManager::handle_jobs() {
expiration_time, wait, false, [tablet, rs,
seg_id](Status st) {
VLOG_DEBUG << "warmup rowset " <<
rs->version() << " segment "
<< seg_id << " completed";
- if (tablet->complete_rowset_segment_warmup(
- rs->rowset_id(), st, 1, 0) ==
WarmUpState::DONE) {
+ if
(tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::JOB,
+
rs->rowset_id(), st, 1,
+ 0)
+ .trigger_source ==
WarmUpTriggerSource::JOB) {
VLOG_DEBUG << "warmup rowset " <<
rs->version() << " completed";
}
});
@@ -264,16 +267,18 @@ void CloudWarmUpManager::handle_jobs() {
}
}
}
-
tablet->update_rowset_warmup_state_inverted_idx_num(rs->rowset_id(), 1);
+
tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::JOB, rs->rowset_id(),
1);
submit_download_tasks(
idx_path, file_size,
storage_resource.value()->fs,
expiration_time, wait, true, [=](Status
st) {
VLOG_DEBUG << "warmup rowset " <<
rs->version()
<< " segment " << seg_id
<< "inverted idx:" <<
idx_path << " completed";
- if
(tablet->complete_rowset_segment_warmup(rs->rowset_id(),
-
st, 0, 1) ==
- WarmUpState::DONE) {
+ if
(tablet->complete_rowset_segment_warmup(
+
WarmUpTriggerSource::JOB, rs->rowset_id(),
+ st, 0, 1)
+ .trigger_source ==
WarmUpTriggerSource::JOB) {
VLOG_DEBUG << "warmup rowset " <<
rs->version()
<< " completed";
}
@@ -285,16 +290,18 @@ void CloudWarmUpManager::handle_jobs() {
storage_resource.value()->remote_idx_v2_path(*rs, seg_id);
file_size = idx_file_info.has_index_size() ?
idx_file_info.index_size()
: -1;
-
tablet->update_rowset_warmup_state_inverted_idx_num(rs->rowset_id(), 1);
+
tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::JOB, rs->rowset_id(),
1);
submit_download_tasks(
idx_path, file_size,
storage_resource.value()->fs,
expiration_time, wait, true, [=](Status
st) {
VLOG_DEBUG << "warmup rowset " <<
rs->version()
<< " segment " << seg_id
<< "inverted idx:" <<
idx_path << " completed";
- if
(tablet->complete_rowset_segment_warmup(rs->rowset_id(),
-
st, 0, 1) ==
- WarmUpState::DONE) {
+ if
(tablet->complete_rowset_segment_warmup(
+
WarmUpTriggerSource::JOB, rs->rowset_id(),
+ st, 0, 1)
+ .trigger_source ==
WarmUpTriggerSource::JOB) {
VLOG_DEBUG << "warmup rowset " <<
rs->version()
<< " completed";
}
diff --git a/be/src/cloud/cloud_warm_up_manager.h
b/be/src/cloud/cloud_warm_up_manager.h
index 73da26b2bfd..ad5f268acb7 100644
--- a/be/src/cloud/cloud_warm_up_manager.h
+++ b/be/src/cloud/cloud_warm_up_manager.h
@@ -38,13 +38,6 @@ enum class DownloadType {
S3,
};
-enum class WarmUpState : int {
- NONE,
- TRIGGERED_BY_SYNC_ROWSET,
- TRIGGERED_BY_JOB,
- DONE,
-};
-
struct JobMeta {
JobMeta() = default;
JobMeta(const TJobMeta& meta);
diff --git a/be/test/cloud/cloud_tablet_test.cpp
b/be/test/cloud/cloud_tablet_test.cpp
index fe9751ff7bf..0e9c39144c6 100644
--- a/be/test/cloud/cloud_tablet_test.cpp
+++ b/be/test/cloud/cloud_tablet_test.cpp
@@ -80,21 +80,24 @@ TEST_F(CloudTabletWarmUpStateTest,
TestGetRowsetWarmupStateNonExistent) {
auto non_existent_id = _engine.next_rowset_id();
WarmUpState state = _tablet->get_rowset_warmup_state(non_existent_id);
- EXPECT_EQ(state, WarmUpState::NONE);
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::NONE,
WarmUpProgress::NONE};
+ EXPECT_EQ(state, expected_state);
}
-// Test add_rowset_warmup_state with TRIGGERED_BY_JOB state
+// Test add_rowset_warmup_state with TRIGGERED_BY_EVENT_DRIVEN state
TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredByJob) {
auto rowset = create_rowset(Version(1, 1), 5);
ASSERT_NE(rowset, nullptr);
bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(result);
// Verify the state is correctly set
WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
}
// Test add_rowset_warmup_state with TRIGGERED_BY_SYNC_ROWSET state
@@ -103,12 +106,14 @@ TEST_F(CloudTabletWarmUpStateTest,
TestAddRowsetWarmupStateTriggeredBySyncRowset
ASSERT_NE(rowset, nullptr);
bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
+
WarmUpTriggerSource::SYNC_ROWSET);
EXPECT_TRUE(result);
// Verify the state is correctly set
WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
}
// Test adding duplicate rowset warmup state should fail
@@ -118,26 +123,29 @@ TEST_F(CloudTabletWarmUpStateTest,
TestAddDuplicateRowsetWarmupState) {
// First addition should succeed
bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(result1);
// Second addition should fail
bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
+
WarmUpTriggerSource::SYNC_ROWSET);
EXPECT_FALSE(result2);
// State should remain the original one
WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
}
// Test complete_rowset_segment_warmup for non-existent rowset
TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupNonExistent)
{
auto non_existent_id = _engine.next_rowset_id();
- WarmUpState result =
- _tablet->complete_rowset_segment_warmup(non_existent_id,
Status::OK(), 1, 0);
- EXPECT_EQ(result, WarmUpState::NONE);
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::SYNC_ROWSET, non_existent_id, Status::OK(),
1, 0);
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::NONE,
WarmUpProgress::NONE};
+ EXPECT_EQ(result, expected_state);
}
// Test complete_rowset_segment_warmup with partial completion
@@ -147,22 +155,26 @@ TEST_F(CloudTabletWarmUpStateTest,
TestCompleteRowsetSegmentWarmupPartial) {
// Add rowset warmup state
bool add_result =
_tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(add_result);
- // Complete one segment, should still be in TRIGGERED_BY_JOB state
- WarmUpState result1 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB);
+ // Complete one segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state
+ WarmUpState result1 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result1, expected_state);
- // Complete second segment, should still be in TRIGGERED_BY_JOB state
- WarmUpState result2 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB);
+ // Complete second segment, should still be in TRIGGERED_BY_EVENT_DRIVEN
state
+ WarmUpState result2 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result2, expected_state);
- // Verify current state is still TRIGGERED_BY_JOB
+ // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN
WarmUpState current_state =
_tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_JOB);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(current_state, expected_state);
}
// Test complete_rowset_segment_warmup with full completion
@@ -172,22 +184,26 @@ TEST_F(CloudTabletWarmUpStateTest,
TestCompleteRowsetSegmentWarmupFull) {
// Add rowset warmup state
bool add_result =
_tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
+
WarmUpTriggerSource::SYNC_ROWSET);
EXPECT_TRUE(add_result);
// Complete first segment
- WarmUpState result1 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
+ WarmUpState result1 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(result1, expected_state);
// Complete second segment, should transition to DONE state
- WarmUpState result2 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- EXPECT_EQ(result2, WarmUpState::DONE);
+ WarmUpState result2 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DONE};
+ EXPECT_EQ(result2, expected_state);
// Verify final state is DONE
WarmUpState final_state =
_tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(final_state, WarmUpState::DONE);
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DONE};
+ EXPECT_EQ(final_state, expected_state);
}
// Test complete_rowset_segment_warmup with inverted index file, partial
completion
@@ -197,25 +213,31 @@ TEST_F(CloudTabletWarmUpStateTest,
TestCompleteRowsetSegmentWarmupWithInvertedIn
// Add rowset warmup state
bool add_result =
_tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(add_result);
- _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(),
1);
- _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(),
1);
+ EXPECT_TRUE(_tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1));
+ EXPECT_TRUE(_tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1));
// Complete one segment file
- WarmUpState result1 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB);
-
- // Complete inverted index file, should still be in TRIGGERED_BY_JOB state
- WarmUpState result2 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 0, 1);
- EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB);
-
- // Verify current state is still TRIGGERED_BY_JOB
+ WarmUpState result1 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result1, expected_state);
+
+ // Complete inverted index file, should still be in
TRIGGERED_BY_EVENT_DRIVEN state
+ WarmUpState result2 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 0, 1);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result2, expected_state);
+
+ // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN
WarmUpState current_state =
_tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_JOB);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(current_state, expected_state);
}
// Test complete_rowset_segment_warmup with inverted index file, full
completion
@@ -225,24 +247,29 @@ TEST_F(CloudTabletWarmUpStateTest,
TestCompleteRowsetSegmentWarmupWithInvertedIn
// Add rowset warmup state
bool add_result =
_tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(add_result);
- _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(),
1);
+ EXPECT_TRUE(_tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1));
// Complete segment file
- WarmUpState result1 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB);
+ WarmUpState result1 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result1, expected_state);
// Complete inverted index file
- WarmUpState result2 =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 0, 1);
- EXPECT_EQ(result2, WarmUpState::DONE);
+ WarmUpState result2 = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 0, 1);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(result2, expected_state);
// Verify final state is DONE
WarmUpState final_state =
_tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(final_state, WarmUpState::DONE);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(final_state, expected_state);
}
// Test complete_rowset_segment_warmup with error status
@@ -252,18 +279,21 @@ TEST_F(CloudTabletWarmUpStateTest,
TestCompleteRowsetSegmentWarmupWithError) {
// Add rowset warmup state
bool add_result =
_tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(add_result);
// Complete with error status, should still transition to DONE when all
segments complete
Status error_status = Status::InternalError("Test error");
- WarmUpState result =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
error_status, 1, 0);
- EXPECT_EQ(result, WarmUpState::DONE);
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
error_status, 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(result, expected_state);
// Verify final state is DONE even with error
WarmUpState final_state =
_tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(final_state, WarmUpState::DONE);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(final_state, expected_state);
}
// Test multiple rowsets warmup state management
@@ -277,35 +307,44 @@ TEST_F(CloudTabletWarmUpStateTest,
TestMultipleRowsetsWarmupState) {
// Add multiple rowsets
EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB));
+
WarmUpTriggerSource::EVENT_DRIVEN));
EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_SYNC_ROWSET));
+
WarmUpTriggerSource::SYNC_ROWSET));
EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset3->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB));
+
WarmUpTriggerSource::EVENT_DRIVEN));
// Verify all states
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()),
- WarmUpState::TRIGGERED_BY_JOB);
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()),
- WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()),
- WarmUpState::TRIGGERED_BY_JOB);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()),
expected_state);
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()),
expected_state);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()),
expected_state);
// Complete rowset1 (2 segments)
- EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(),
Status::OK(), 1, 0),
- WarmUpState::TRIGGERED_BY_JOB);
- EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(),
Status::OK(), 1, 0),
- WarmUpState::DONE);
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result, expected_state);
+ result =
_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN,
+ rowset1->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(result, expected_state);
// Complete rowset3 (1 segment)
- EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset3->rowset_id(),
Status::OK(), 1, 0),
- WarmUpState::DONE);
+ result =
_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN,
+ rowset3->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(result, expected_state);
// Verify states after completion
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()),
WarmUpState::DONE);
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()),
- WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()),
WarmUpState::DONE);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()),
expected_state);
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()),
expected_state);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()),
expected_state);
}
// Test warmup state with zero segments (edge case)
@@ -315,18 +354,14 @@ TEST_F(CloudTabletWarmUpStateTest,
TestWarmupStateWithZeroSegments) {
// Add rowset with zero segments
bool add_result =
_tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB);
+
WarmUpTriggerSource::EVENT_DRIVEN);
EXPECT_TRUE(add_result);
// State should be immediately ready for completion since there are no
segments to warm up
WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
- EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB);
-
- // Any completion call should handle the edge case gracefully
- WarmUpState result =
- _tablet->complete_rowset_segment_warmup(rowset->rowset_id(),
Status::OK(), 1, 0);
- // With 0 segments, the counter should already be 0, so this should
transition to DONE
- EXPECT_EQ(result, WarmUpState::DONE);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(state, expected_state);
}
// Test concurrent access to warmup state (basic thread safety verification)
@@ -338,22 +373,304 @@ TEST_F(CloudTabletWarmUpStateTest,
TestConcurrentWarmupStateAccess) {
// Add rowsets from different "threads" (simulated by sequential calls)
EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_JOB));
+
WarmUpTriggerSource::EVENT_DRIVEN));
EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()),
-
WarmUpState::TRIGGERED_BY_SYNC_ROWSET));
+
WarmUpTriggerSource::SYNC_ROWSET));
// Interleaved completion operations
- EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(),
Status::OK(), 1, 0),
- WarmUpState::TRIGGERED_BY_JOB);
- EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset2->rowset_id(),
Status::OK(), 1, 0),
- WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
- EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(),
Status::OK(), 1, 0),
- WarmUpState::TRIGGERED_BY_JOB);
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result, expected_state);
+ result =
_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET,
+ rowset2->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(result, expected_state);
+ result =
_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN,
+ rowset1->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(result, expected_state);
// Check states are maintained correctly
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()),
- WarmUpState::TRIGGERED_BY_JOB);
- EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()),
- WarmUpState::TRIGGERED_BY_SYNC_ROWSET);
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()),
expected_state);
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()),
expected_state);
+}
+
+// Test only when the trigger source matches can the state be updated
+TEST_F(CloudTabletWarmUpStateTest,
TestCompleteRowsetSegmentWarmupTriggerSource) {
+ auto rowset = create_rowset(Version(13, 13), 1);
+ ASSERT_NE(rowset, nullptr);
+ // Add rowset warmup state
+ bool add_result =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(add_result);
+
+ // Attempt to update inverted index num with a different trigger source,
should fail
+ bool update_result = _tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), 1);
+ EXPECT_FALSE(update_result);
+ update_result = _tablet->update_rowset_warmup_state_inverted_idx_num(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1);
+ EXPECT_FALSE(update_result);
+
+ // Attempt to complete with a different trigger source, should not update
state
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(result, expected_state);
+ result =
_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN,
+ rowset->rowset_id(),
Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(result, expected_state);
+
+ // Now complete with the correct trigger source
+ result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::JOB,
rowset->rowset_id(),
+ Status::OK(), 1, 0);
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DONE};
+ EXPECT_EQ(result, expected_state);
+}
+
+// Test JOB trigger source can override non-JOB warmup states
+TEST_F(CloudTabletWarmUpStateTest, TestJobTriggerOverridesNonJobStates) {
+ auto rowset = create_rowset(Version(14, 14), 2);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add EVENT_DRIVEN warmup state
+ bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::EVENT_DRIVEN);
+ EXPECT_TRUE(result1);
+
+ // Verify EVENT_DRIVEN state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Now add JOB warmup state, should override EVENT_DRIVEN
+ bool result2 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result2);
+
+ // Verify JOB state has overridden EVENT_DRIVEN
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test JOB trigger source can override SYNC_ROWSET warmup state
+TEST_F(CloudTabletWarmUpStateTest, TestJobTriggerOverridesSyncRowsetState) {
+ auto rowset = create_rowset(Version(15, 15), 3);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add SYNC_ROWSET warmup state
+ bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::SYNC_ROWSET);
+ EXPECT_TRUE(result1);
+
+ // Verify SYNC_ROWSET state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Now add JOB warmup state, should override SYNC_ROWSET
+ bool result2 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result2);
+
+ // Verify JOB state has overridden SYNC_ROWSET
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test JOB trigger source cannot override another JOB warmup state
+TEST_F(CloudTabletWarmUpStateTest, TestJobTriggerCannotOverrideAnotherJob) {
+ auto rowset = create_rowset(Version(16, 16), 2);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add JOB warmup state
+ bool result1 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result1);
+
+ // Verify first JOB state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Try to add another JOB warmup state, should fail
+ bool result2 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_FALSE(result2);
+
+ // Verify state remains the original JOB state
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test EVENT_DRIVEN cannot override existing JOB state
+TEST_F(CloudTabletWarmUpStateTest, TestEventDrivenCannotOverrideJobState) {
+ auto rowset = create_rowset(Version(17, 17), 1);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add JOB warmup state
+ bool result1 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result1);
+
+ // Verify JOB state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Try to add EVENT_DRIVEN warmup state, should fail
+ bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::EVENT_DRIVEN);
+ EXPECT_FALSE(result2);
+
+ // Verify state remains JOB
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test SYNC_ROWSET cannot override existing JOB state
+TEST_F(CloudTabletWarmUpStateTest, TestSyncRowsetCannotOverrideJobState) {
+ auto rowset = create_rowset(Version(18, 18), 2);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add JOB warmup state
+ bool result1 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result1);
+
+ // Verify JOB state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Try to add SYNC_ROWSET warmup state, should fail
+ bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::SYNC_ROWSET);
+ EXPECT_FALSE(result2);
+
+ // Verify state remains JOB
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test EVENT_DRIVEN cannot override existing SYNC_ROWSET state
+TEST_F(CloudTabletWarmUpStateTest,
TestEventDrivenCannotOverrideSyncRowsetState) {
+ auto rowset = create_rowset(Version(19, 19), 1);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add SYNC_ROWSET warmup state
+ bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::SYNC_ROWSET);
+ EXPECT_TRUE(result1);
+
+ // Verify SYNC_ROWSET state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Try to add EVENT_DRIVEN warmup state, should fail
+ bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::EVENT_DRIVEN);
+ EXPECT_FALSE(result2);
+
+ // Verify state remains SYNC_ROWSET
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test SYNC_ROWSET cannot override existing EVENT_DRIVEN state
+TEST_F(CloudTabletWarmUpStateTest,
TestSyncRowsetCannotOverrideEventDrivenState) {
+ auto rowset = create_rowset(Version(20, 20), 3);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add EVENT_DRIVEN warmup state
+ bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::EVENT_DRIVEN);
+ EXPECT_TRUE(result1);
+
+ // Verify EVENT_DRIVEN state is set
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+
+ // Try to add SYNC_ROWSET warmup state, should fail
+ bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::SYNC_ROWSET);
+ EXPECT_FALSE(result2);
+
+ // Verify state remains EVENT_DRIVEN
+ state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test JOB can override DONE state from non-JOB source
+TEST_F(CloudTabletWarmUpStateTest, TestJobCanOverrideDoneStateFromNonJob) {
+ auto rowset = create_rowset(Version(21, 21), 1);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add EVENT_DRIVEN warmup state
+ bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
+
WarmUpTriggerSource::EVENT_DRIVEN);
+ EXPECT_TRUE(result1);
+
+ // Complete the warmup to DONE state
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(),
Status::OK(), 1, 0);
+ WarmUpState expected_state =
+ WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN,
WarmUpProgress::DONE};
+ EXPECT_EQ(result, expected_state);
+
+ // Now add JOB warmup state, should override the DONE state
+ bool result2 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result2);
+
+ // Verify JOB state has overridden the DONE state
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
+}
+
+// Test JOB can override DONE state from JOB source
+TEST_F(CloudTabletWarmUpStateTest, TestJobCanOverrideDoneStateFromJob) {
+ auto rowset = create_rowset(Version(21, 21), 1);
+ ASSERT_NE(rowset, nullptr);
+
+ // First, add EVENT_DRIVEN warmup state
+ bool result1 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result1);
+
+ // Complete the warmup to DONE state
+ WarmUpState result = _tablet->complete_rowset_segment_warmup(
+ WarmUpTriggerSource::JOB, rowset->rowset_id(), Status::OK(), 1, 0);
+ WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DONE};
+ EXPECT_EQ(result, expected_state);
+
+ // Now add JOB warmup state, should override the DONE state
+ bool result2 =
+ _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()),
WarmUpTriggerSource::JOB);
+ EXPECT_TRUE(result2);
+
+ // Verify JOB state has overridden the DONE state
+ WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id());
+ expected_state = WarmUpState {WarmUpTriggerSource::JOB,
WarmUpProgress::DOING};
+ EXPECT_EQ(state, expected_state);
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
