This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new b6662a5d15d branch-3.0-pick: [Opt](profile) Add profile for
`sync_rowsets` (#49864) (#49926)
b6662a5d15d is described below
commit b6662a5d15dd60742095948c53303c4dc27df964
Author: bobhan1 <bao...@selectdb.com>
AuthorDate: Fri Apr 11 10:13:03 2025 +0800
branch-3.0-pick: [Opt](profile) Add profile for `sync_rowsets` (#49864)
(#49926)
pick https://github.com/apache/doris/pull/49864
---
be/src/cloud/cloud_meta_mgr.cpp | 28 ++++++++++--
be/src/cloud/cloud_meta_mgr.h | 5 +-
be/src/cloud/cloud_storage_engine.cpp | 5 +-
be/src/cloud/cloud_storage_engine.h | 3 +-
be/src/cloud/cloud_tablet.cpp | 11 +++--
be/src/cloud/cloud_tablet.h | 16 ++++++-
be/src/cloud/cloud_tablet_mgr.cpp | 9 ++--
be/src/cloud/cloud_tablet_mgr.h | 4 +-
be/src/olap/storage_engine.cpp | 2 +-
be/src/olap/storage_engine.h | 6 ++-
be/src/pipeline/exec/olap_scan_operator.cpp | 71 ++++++++++++++++++++++++++---
be/src/pipeline/exec/olap_scan_operator.h | 7 +++
be/src/runtime/exec_env.cpp | 4 +-
be/src/runtime/exec_env.h | 4 +-
14 files changed, 142 insertions(+), 33 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 6f6024f1912..b5ee1585400 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -439,7 +439,8 @@ Status CloudMetaMgr::get_tablet_meta(int64_t tablet_id,
TabletMetaSharedPtr* tab
}
Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool
warmup_delta_data,
- bool sync_delete_bitmap, bool
full_sync) {
+ bool sync_delete_bitmap, bool
full_sync,
+ SyncRowsetStats* sync_stats) {
using namespace std::chrono;
TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets",
Status::OK(), tablet);
@@ -524,6 +525,11 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet*
tablet, bool warmup_delta_
int64_t now =
duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
tablet->last_sync_time_s = now;
+ if (sync_stats) {
+ sync_stats->get_remote_rowsets_rpc_ms += latency / 1000;
+ sync_stats->get_remote_rowsets_num += resp.rowset_meta().size();
+ }
+
// If is mow, the tablet has no delete bitmap in base rowsets.
// So dont need to sync it.
if (sync_delete_bitmap && tablet->enable_unique_key_merge_on_write() &&
@@ -531,7 +537,8 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet*
tablet, bool warmup_delta_
DeleteBitmap delete_bitmap(tablet_id);
int64_t old_max_version = req.start_version() - 1;
auto st = sync_tablet_delete_bitmap(tablet, old_max_version,
resp.rowset_meta(),
- resp.stats(), req.idx(),
&delete_bitmap, full_sync);
+ resp.stats(), req.idx(),
&delete_bitmap, full_sync,
+ sync_stats);
if (st.is<ErrorCode::ROWSETS_EXPIRED>() && tried++ < retry_times) {
LOG_WARNING("rowset meta is expired, need to retry")
.tag("tablet", tablet->tablet_id())
@@ -698,13 +705,17 @@ bool
CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64
Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t
old_max_version,
std::ranges::range auto&&
rs_metas,
const TabletStatsPB& stats,
const TabletIndexPB& idx,
- DeleteBitmap* delete_bitmap,
bool full_sync) {
+ DeleteBitmap* delete_bitmap,
bool full_sync,
+ SyncRowsetStats* sync_stats) {
if (rs_metas.empty()) {
return Status::OK();
}
if (!full_sync &&
sync_tablet_delete_bitmap_by_cache(tablet, old_max_version, rs_metas,
delete_bitmap)) {
+ if (sync_stats) {
+ sync_stats->get_local_delete_bitmap_rowsets_num += rs_metas.size();
+ }
return Status::OK();
} else {
DeleteBitmapPtr new_delete_bitmap =
std::make_shared<DeleteBitmap>(tablet->tablet_id());
@@ -740,6 +751,9 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet*
tablet, int64_t old_
req.add_end_versions(new_max_version);
}
}
+ if (sync_stats) {
+ sync_stats->get_remote_delete_bitmap_rowsets_num +=
req.rowset_ids_size();
+ }
VLOG_DEBUG << "send GetDeleteBitmapRequest: " << req.ShortDebugString();
@@ -792,6 +806,14 @@ Status
CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_
"rowset_ids.size={},segment_ids.size={},vers.size={},delete_bitmaps.size={}",
rowset_ids.size(), segment_ids.size(), vers.size(),
delete_bitmaps.size());
}
+ if (sync_stats) {
+ sync_stats->get_remote_delete_bitmap_rpc_ms +=
+ std::chrono::duration_cast<std::chrono::milliseconds>(end -
start).count();
+ sync_stats->get_remote_delete_bitmap_key_count +=
delete_bitmaps.size();
+ for (const auto& dbm : delete_bitmaps) {
+ sync_stats->get_remote_delete_bitmap_bytes += dbm.length();
+ }
+ }
for (int i = 0; i < rowset_ids.size(); i++) {
RowsetId rst_id;
rst_id.init(rowset_ids[i]);
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index a666a5e4d16..3ff817a8710 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -63,7 +63,8 @@ public:
Status get_schema_dict(int64_t index_id,
std::shared_ptr<SchemaCloudDictionary>* schema_dict);
Status sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_data =
false,
- bool sync_delete_bitmap = true, bool full_sync
= false);
+ bool sync_delete_bitmap = true, bool full_sync
= false,
+ SyncRowsetStats* sync_stats = nullptr);
Status prepare_rowset(const RowsetMeta& rs_meta,
std::shared_ptr<RowsetMeta>* existed_rs_meta =
nullptr);
@@ -123,7 +124,7 @@ private:
Status sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t
old_max_version,
std::ranges::range auto&& rs_metas, const
TabletStatsPB& stats,
const TabletIndexPB& idx, DeleteBitmap*
delete_bitmap,
- bool full_sync = false);
+ bool full_sync = false, SyncRowsetStats*
sync_stats = nullptr);
void check_table_size_correctness(const RowsetMeta& rs_meta);
int64_t get_segment_file_size(const RowsetMeta& rs_meta);
int64_t get_inverted_index_file_szie(const RowsetMeta& rs_meta);
diff --git a/be/src/cloud/cloud_storage_engine.cpp
b/be/src/cloud/cloud_storage_engine.cpp
index 45d53642140..b6498a0080d 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -236,8 +236,9 @@ bool CloudStorageEngine::stopped() {
return _stopped;
}
-Result<BaseTabletSPtr> CloudStorageEngine::get_tablet(int64_t tablet_id) {
- return _tablet_mgr->get_tablet(tablet_id, false).transform([](auto&& t) {
+Result<BaseTabletSPtr> CloudStorageEngine::get_tablet(int64_t tablet_id,
+ SyncRowsetStats*
sync_stats) {
+ return _tablet_mgr->get_tablet(tablet_id, false, true,
sync_stats).transform([](auto&& t) {
return static_pointer_cast<BaseTablet>(std::move(t));
});
}
diff --git a/be/src/cloud/cloud_storage_engine.h
b/be/src/cloud/cloud_storage_engine.h
index 046116e3a24..b50f62ac06d 100644
--- a/be/src/cloud/cloud_storage_engine.h
+++ b/be/src/cloud/cloud_storage_engine.h
@@ -57,7 +57,8 @@ public:
void stop() override;
bool stopped() override;
- Result<BaseTabletSPtr> get_tablet(int64_t tablet_id) override;
+ Result<BaseTabletSPtr> get_tablet(int64_t tablet_id,
+ SyncRowsetStats* sync_stats = nullptr)
override;
Status start_bg_threads() override;
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index cf3fe051bec..61d6fa59bda 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -144,8 +144,9 @@ Status CloudTablet::merge_rowsets_schema() {
// There are only two tablet_states RUNNING and NOT_READY in cloud mode
// This function will erase the tablet from `CloudTabletMgr` when it can't
find this tablet in MS.
-Status CloudTablet::sync_rowsets(int64_t query_version, bool
warmup_delta_data) {
- RETURN_IF_ERROR(sync_if_not_running());
+Status CloudTablet::sync_rowsets(int64_t query_version, bool warmup_delta_data,
+ SyncRowsetStats* stats) {
+ RETURN_IF_ERROR(sync_if_not_running(stats));
if (query_version > 0) {
std::shared_lock rlock(_meta_lock);
@@ -163,7 +164,7 @@ Status CloudTablet::sync_rowsets(int64_t query_version,
bool warmup_delta_data)
}
}
- auto st = _engine.meta_mgr().sync_tablet_rowsets(this, warmup_delta_data);
+ auto st = _engine.meta_mgr().sync_tablet_rowsets(this, warmup_delta_data,
true, false, stats);
if (st.is<ErrorCode::NOT_FOUND>()) {
clear_cache();
}
@@ -174,7 +175,7 @@ Status CloudTablet::sync_rowsets(int64_t query_version,
bool warmup_delta_data)
// Sync tablet meta and all rowset meta if not running.
// This could happen when BE didn't finish schema change job and another BE
committed this schema change job.
// It should be a quite rare situation.
-Status CloudTablet::sync_if_not_running() {
+Status CloudTablet::sync_if_not_running(SyncRowsetStats* stats) {
if (tablet_state() == TABLET_RUNNING) {
return Status::OK();
}
@@ -215,7 +216,7 @@ Status CloudTablet::sync_if_not_running() {
_max_version = -1;
}
- st = _engine.meta_mgr().sync_tablet_rowsets(this);
+ st = _engine.meta_mgr().sync_tablet_rowsets(this, false, true, false,
stats);
if (st.is<ErrorCode::NOT_FOUND>()) {
clear_cache();
}
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index 4226e26a0f5..baa6fdd8e1b 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -26,6 +26,17 @@ namespace doris {
class CloudStorageEngine;
+struct SyncRowsetStats {
+ int64_t get_remote_rowsets_num {0};
+ int64_t get_remote_rowsets_rpc_ms {0};
+
+ int64_t get_local_delete_bitmap_rowsets_num {0};
+ int64_t get_remote_delete_bitmap_rowsets_num {0};
+ int64_t get_remote_delete_bitmap_key_count {0};
+ int64_t get_remote_delete_bitmap_bytes {0};
+ int64_t get_remote_delete_bitmap_rpc_ms {0};
+};
+
class CloudTablet final : public BaseTablet {
public:
CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta);
@@ -68,7 +79,8 @@ public:
// If `query_version` > 0 and local max_version of the tablet >=
`query_version`, do nothing.
// If 'need_download_data_async' is true, it means that we need to
download the new version
// rowsets datum async.
- Status sync_rowsets(int64_t query_version = -1, bool warmup_delta_data =
false);
+ Status sync_rowsets(int64_t query_version = -1, bool warmup_delta_data =
false,
+ SyncRowsetStats* stats = nullptr);
// Synchronize the tablet meta from meta service.
Status sync_meta();
@@ -219,7 +231,7 @@ private:
// FIXME(plat1ko): No need to record base size if rowsets are ordered by
version
void update_base_size(const Rowset& rs);
- Status sync_if_not_running();
+ Status sync_if_not_running(SyncRowsetStats* stats = nullptr);
CloudStorageEngine& _engine;
diff --git a/be/src/cloud/cloud_tablet_mgr.cpp
b/be/src/cloud/cloud_tablet_mgr.cpp
index 04a1c33d5c3..23259547fc9 100644
--- a/be/src/cloud/cloud_tablet_mgr.cpp
+++ b/be/src/cloud/cloud_tablet_mgr.cpp
@@ -150,7 +150,8 @@ void set_tablet_access_time_ms(CloudTablet* tablet) {
}
Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t
tablet_id, bool warmup_data,
- bool
sync_delete_bitmap) {
+ bool
sync_delete_bitmap,
+
SyncRowsetStats* sync_stats) {
// LRU value type. `Value`'s lifetime MUST NOT be longer than
`CloudTabletMgr`
class Value : public LRUCacheValueBase {
public:
@@ -168,8 +169,8 @@ Result<std::shared_ptr<CloudTablet>>
CloudTabletMgr::get_tablet(int64_t tablet_i
CacheKey key(tablet_id_str);
auto* handle = _cache->lookup(key);
if (handle == nullptr) {
- auto load_tablet = [this, &key, warmup_data,
- sync_delete_bitmap](int64_t tablet_id) ->
std::shared_ptr<CloudTablet> {
+ auto load_tablet = [this, &key, warmup_data, sync_delete_bitmap,
+ sync_stats](int64_t tablet_id) ->
std::shared_ptr<CloudTablet> {
TabletMetaSharedPtr tablet_meta;
auto st = _engine.meta_mgr().get_tablet_meta(tablet_id,
&tablet_meta);
if (!st.ok()) {
@@ -181,7 +182,7 @@ Result<std::shared_ptr<CloudTablet>>
CloudTabletMgr::get_tablet(int64_t tablet_i
auto value = std::make_unique<Value>(tablet, *_tablet_map);
// MUST sync stats to let compaction scheduler work correctly
st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(),
warmup_data,
- sync_delete_bitmap);
+ sync_delete_bitmap,
false, sync_stats);
if (!st.ok()) {
LOG(WARNING) << "failed to sync tablet " << tablet_id << ": "
<< st;
return nullptr;
diff --git a/be/src/cloud/cloud_tablet_mgr.h b/be/src/cloud/cloud_tablet_mgr.h
index 409c2eb35a0..1a6ec72c1f7 100644
--- a/be/src/cloud/cloud_tablet_mgr.h
+++ b/be/src/cloud/cloud_tablet_mgr.h
@@ -33,6 +33,7 @@ class CloudTablet;
class CloudStorageEngine;
class LRUCachePolicy;
class CountDownLatch;
+struct SyncRowsetStats;
extern uint64_t g_tablet_report_inactive_duration_ms;
@@ -44,7 +45,8 @@ public:
// If the tablet is in cache, return this tablet directly; otherwise will
get tablet meta first,
// sync rowsets after, and download segment data in background if
`warmup_data` is true.
Result<std::shared_ptr<CloudTablet>> get_tablet(int64_t tablet_id, bool
warmup_data = false,
- bool sync_delete_bitmap =
true);
+ bool sync_delete_bitmap =
true,
+ SyncRowsetStats*
sync_stats = nullptr);
void erase_tablet(int64_t tablet_id);
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 84a654e366d..2ece51324f3 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -1268,7 +1268,7 @@ Status StorageEngine::create_tablet(const
TCreateTabletReq& request, RuntimeProf
return _tablet_manager->create_tablet(request, stores, profile);
}
-Result<BaseTabletSPtr> StorageEngine::get_tablet(int64_t tablet_id) {
+Result<BaseTabletSPtr> StorageEngine::get_tablet(int64_t tablet_id,
SyncRowsetStats* sync_stats) {
BaseTabletSPtr tablet;
std::string err;
tablet = _tablet_manager->get_tablet(tablet_id, true, &err);
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index e5a48f436e7..7d7ec9d1190 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -108,7 +108,8 @@ public:
// start all background threads. This should be call after env is ready.
virtual Status start_bg_threads() = 0;
- virtual Result<BaseTabletSPtr> get_tablet(int64_t tablet_id) = 0;
+ virtual Result<BaseTabletSPtr> get_tablet(int64_t tablet_id,
+ SyncRowsetStats* sync_stats =
nullptr) = 0;
void register_report_listener(ReportWorker* listener);
void deregister_report_listener(ReportWorker* listener);
@@ -220,7 +221,8 @@ public:
Status create_tablet(const TCreateTabletReq& request, RuntimeProfile*
profile);
- Result<BaseTabletSPtr> get_tablet(int64_t tablet_id) override;
+ Result<BaseTabletSPtr> get_tablet(int64_t tablet_id,
+ SyncRowsetStats* sync_stats = nullptr)
override;
void clear_transaction_task(const TTransactionId transaction_id);
void clear_transaction_task(const TTransactionId transaction_id,
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index e85d4ac14a4..e3d112c2b07 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -33,6 +33,7 @@
#include "pipeline/exec/scan_operator.h"
#include "pipeline/query_cache/query_cache.h"
#include "service/backend_options.h"
+#include "util/runtime_profile.h"
#include "util/to_string.h"
#include "vec/exec/scan/new_olap_scanner.h"
#include "vec/exprs/vectorized_fn_call.h"
@@ -64,7 +65,29 @@ Status OlapScanLocalState::_init_profile() {
_block_fetch_timer = ADD_TIMER(_scanner_profile, "BlockFetchTime");
_delete_bitmap_get_agg_timer = ADD_TIMER(_scanner_profile,
"DeleteBitmapGetAggTime");
if (config::is_cloud_mode()) {
- _sync_rowset_timer = ADD_TIMER(_scanner_profile, "SyncRowsetTime");
+ static const char* sync_rowset_timer_name = "SyncRowsetTime";
+ _sync_rowset_timer = ADD_TIMER(_scanner_profile,
sync_rowset_timer_name);
+ _sync_rowset_get_remote_rowsets_num =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetRemoteRowsetsCount", TUnit::UNIT,
+ sync_rowset_timer_name);
+ _sync_rowset_get_remote_rowsets_rpc_timer =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetRemoteRowsetsRpcMs",
+ TUnit::TIME_MS, sync_rowset_timer_name);
+ _sync_rowset_get_local_delete_bitmap_rowsets_num =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetLocalDeleteBitmapRowsetsCount",
+ TUnit::UNIT, sync_rowset_timer_name);
+ _sync_rowset_get_remote_delete_bitmap_rowsets_num =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetRemoteDeleteBitmapRowsetsCount",
+ TUnit::UNIT, sync_rowset_timer_name);
+ _sync_rowset_get_remote_delete_bitmap_key_count =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetRemoteDeleteBitmapKeyCount",
+ TUnit::UNIT, sync_rowset_timer_name);
+ _sync_rowset_get_remote_delete_bitmap_bytes =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetRemoteDeleteBitmapBytes",
+ TUnit::BYTES, sync_rowset_timer_name);
+ _sync_rowset_get_remote_delete_bitmap_rpc_timer =
+ ADD_CHILD_COUNTER(_scanner_profile,
"SyncRowsetGetRemoteDeleteBitmapRpcMs",
+ TUnit::TIME_MS, sync_rowset_timer_name);
}
_block_init_timer = ADD_TIMER(_segment_profile, "BlockInitTime");
_block_init_seek_timer = ADD_TIMER(_segment_profile, "BlockInitSeekTime");
@@ -415,6 +438,23 @@ Status OlapScanLocalState::hold_tablets() {
if (!_tablets.empty()) {
return Status::OK();
}
+
+ auto update_sync_rowset_profile = [&](const SyncRowsetStats& sync_stat) {
+ COUNTER_UPDATE(_sync_rowset_get_remote_rowsets_num,
sync_stat.get_remote_rowsets_num);
+ COUNTER_UPDATE(_sync_rowset_get_remote_rowsets_rpc_timer,
+ sync_stat.get_remote_rowsets_rpc_ms);
+ COUNTER_UPDATE(_sync_rowset_get_local_delete_bitmap_rowsets_num,
+ sync_stat.get_local_delete_bitmap_rowsets_num);
+ COUNTER_UPDATE(_sync_rowset_get_remote_delete_bitmap_rowsets_num,
+ sync_stat.get_remote_delete_bitmap_rowsets_num);
+ COUNTER_UPDATE(_sync_rowset_get_remote_delete_bitmap_key_count,
+ sync_stat.get_remote_delete_bitmap_key_count);
+ COUNTER_UPDATE(_sync_rowset_get_remote_delete_bitmap_bytes,
+ sync_stat.get_remote_delete_bitmap_bytes);
+ COUNTER_UPDATE(_sync_rowset_get_remote_delete_bitmap_rpc_timer,
+ sync_stat.get_remote_delete_bitmap_rpc_ms);
+ };
+
MonotonicStopWatch timer;
timer.start();
_tablets.resize(_scan_ranges.size());
@@ -423,31 +463,48 @@ Status OlapScanLocalState::hold_tablets() {
int64_t version = 0;
std::from_chars(_scan_ranges[i]->version.data(),
_scan_ranges[i]->version.data() +
_scan_ranges[i]->version.size(), version);
- auto tablet =
DORIS_TRY(ExecEnv::get_tablet(_scan_ranges[i]->tablet_id));
- _tablets[i] = {std::move(tablet), version};
-
if (config::is_cloud_mode()) {
+ int64_t duration_ns = 0;
+ SyncRowsetStats sync_stats;
+ {
+ SCOPED_RAW_TIMER(&duration_ns);
+ auto tablet =
+
DORIS_TRY(ExecEnv::get_tablet(_scan_ranges[i]->tablet_id, &sync_stats));
+ _tablets[i] = {std::move(tablet), version};
+ }
+ COUNTER_UPDATE(_sync_rowset_timer, duration_ns);
+ update_sync_rowset_profile(sync_stats);
+
// FIXME(plat1ko): Avoid pointer cast
ExecEnv::GetInstance()->storage_engine().to_cloud().tablet_hotspot().count(
*_tablets[i].tablet);
+ } else {
+ auto tablet =
DORIS_TRY(ExecEnv::get_tablet(_scan_ranges[i]->tablet_id));
+ _tablets[i] = {std::move(tablet), version};
}
}
if (config::is_cloud_mode()) {
int64_t duration_ns = 0;
+ std::vector<SyncRowsetStats> sync_statistics {};
+ sync_statistics.reserve(_tablets.size());
{
SCOPED_RAW_TIMER(&duration_ns);
std::vector<std::function<Status()>> tasks;
tasks.reserve(_scan_ranges.size());
for (auto&& [cur_tablet, cur_version] : _tablets) {
- tasks.emplace_back([cur_tablet, cur_version]() {
+ sync_statistics.emplace_back();
+ tasks.emplace_back([cur_tablet, cur_version, stats =
&sync_statistics.back()]() {
return std::dynamic_pointer_cast<CloudTablet>(cur_tablet)
- ->sync_rowsets(cur_version);
+ ->sync_rowsets(cur_version, false, stats);
});
}
RETURN_IF_ERROR(cloud::bthread_fork_join(tasks, 10));
}
- _sync_rowset_timer->update(duration_ns);
+ COUNTER_UPDATE(_sync_rowset_timer, duration_ns);
+ for (const auto& sync_stats : sync_statistics) {
+ update_sync_rowset_profile(sync_stats);
+ }
}
for (size_t i = 0; i < _scan_ranges.size(); i++) {
RETURN_IF_ERROR(_tablets[i].tablet->capture_rs_readers(
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index f065cd881cd..0b1d015bc22 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -131,6 +131,13 @@ private:
RuntimeProfile::Counter* _block_fetch_timer = nullptr;
RuntimeProfile::Counter* _delete_bitmap_get_agg_timer = nullptr;
RuntimeProfile::Counter* _sync_rowset_timer = nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_remote_rowsets_num = nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_remote_rowsets_rpc_timer =
nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_local_delete_bitmap_rowsets_num
= nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_remote_delete_bitmap_rowsets_num
= nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_remote_delete_bitmap_key_count =
nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_remote_delete_bitmap_bytes =
nullptr;
+ RuntimeProfile::Counter* _sync_rowset_get_remote_delete_bitmap_rpc_timer =
nullptr;
RuntimeProfile::Counter* _block_load_timer = nullptr;
RuntimeProfile::Counter* _block_load_counter = nullptr;
// Add more detail seek timer and counter profile
diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp
index ab24d7ca192..afd946afc05 100644
--- a/be/src/runtime/exec_env.cpp
+++ b/be/src/runtime/exec_env.cpp
@@ -57,10 +57,10 @@ void ExecEnv::set_write_cooldown_meta_executors() {
}
#endif // BE_TEST
-Result<BaseTabletSPtr> ExecEnv::get_tablet(int64_t tablet_id) {
+Result<BaseTabletSPtr> ExecEnv::get_tablet(int64_t tablet_id, SyncRowsetStats*
sync_stats) {
auto storage_engine = GetInstance()->_storage_engine.get();
return storage_engine != nullptr
- ? storage_engine->get_tablet(tablet_id)
+ ? storage_engine->get_tablet(tablet_id, sync_stats)
: ResultError(Status::InternalError("failed to get tablet
{}", tablet_id));
}
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index f891002355d..35b0c842d70 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -114,6 +114,7 @@ class ProcessProfile;
class HeapProfiler;
class WalManager;
class DNSCache;
+struct SyncRowsetStats;
inline bool k_doris_exit = false;
@@ -146,7 +147,8 @@ public:
}
// Requires ExenEnv ready
- static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id);
+ static Result<BaseTabletSPtr> get_tablet(int64_t tablet_id,
+ SyncRowsetStats* sync_stats =
nullptr);
static bool ready() { return _s_ready.load(std::memory_order_acquire); }
static bool tracking_memory() { return
_s_tracking_memory.load(std::memory_order_acquire); }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
