This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 73d47885e8c [enhance](cloud)add log for compaction read time (#51355)
73d47885e8c is described below
commit 73d47885e8cd4b9b30e549105045b72b04e1153a
Author: koarz <[email protected]>
AuthorDate: Tue Jun 17 15:38:08 2025 +0800
[enhance](cloud)add log for compaction read time (#51355)
Problem Summary:
curl http://ip:8040/api/compaction/show?tablet_id=1748942349697
```text
{
......
"exec compaction time us": "217890",
"compaction local read time us": "730",
"compaction remote read time us": "0",
......
}
```
---
be/src/cloud/cloud_base_compaction.cpp | 2 ++
be/src/cloud/cloud_cumulative_compaction.cpp | 2 ++
be/src/cloud/cloud_full_compaction.cpp | 2 ++
be/src/cloud/cloud_tablet.cpp | 16 ++++++++++++++++
be/src/cloud/cloud_tablet.h | 4 ++++
be/src/olap/compaction.cpp | 7 +++++++
be/src/olap/merger.cpp | 13 +++++++++++++
be/src/olap/merger.h | 2 ++
8 files changed, 48 insertions(+)
diff --git a/be/src/cloud/cloud_base_compaction.cpp
b/be/src/cloud/cloud_base_compaction.cpp
index 682f7e158dd..99b916b45e5 100644
--- a/be/src/cloud/cloud_base_compaction.cpp
+++ b/be/src/cloud/cloud_base_compaction.cpp
@@ -303,6 +303,8 @@ Status CloudBaseCompaction::execute_compact() {
.tag("output_rowset_data_size", _output_rowset->data_disk_size())
.tag("output_rowset_index_size", _output_rowset->index_disk_size())
.tag("output_rowset_total_size", _output_rowset->total_disk_size())
+ .tag("local_read_time_us", _stats.cloud_local_read_time)
+ .tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index ed696c8b63d..81c8f78f613 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -226,6 +226,8 @@ Status CloudCumulativeCompaction::execute_compact() {
.tag("cumulative_point", cloud_tablet()->cumulative_layer_point())
.tag("num_rowsets",
cloud_tablet()->fetch_add_approximate_num_rowsets(0))
.tag("cumu_num_rowsets",
cloud_tablet()->fetch_add_approximate_cumu_num_rowsets(0))
+ .tag("local_read_time_us", _stats.cloud_local_read_time)
+ .tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);
diff --git a/be/src/cloud/cloud_full_compaction.cpp
b/be/src/cloud/cloud_full_compaction.cpp
index 3508b94540c..8aadcf9ffe0 100644
--- a/be/src/cloud/cloud_full_compaction.cpp
+++ b/be/src/cloud/cloud_full_compaction.cpp
@@ -183,6 +183,8 @@ Status CloudFullCompaction::execute_compact() {
.tag("output_rowset_data_size", _output_rowset->data_disk_size())
.tag("output_rowset_index_size", _output_rowset->index_disk_size())
.tag("output_rowset_total_size", _output_rowset->total_disk_size())
+ .tag("local_read_time_us", _stats.cloud_local_read_time)
+ .tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 2b86d76cf31..df90408e244 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -18,6 +18,7 @@
#include "cloud/cloud_tablet.h"
#include <bvar/bvar.h>
+#include <gen_cpp/Types_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <rapidjson/document.h>
#include <rapidjson/encodings.h>
@@ -798,6 +799,21 @@ void CloudTablet::get_compaction_status(std::string*
json_result) {
cast_set<uint>(_last_full_compaction_status.length()),
root.GetAllocator());
root.AddMember("last full status", full_compaction_status_value,
root.GetAllocator());
+ rapidjson::Value exec_compaction_time;
+ std::string num_str {std::to_string(exec_compaction_time_us.load())};
+ exec_compaction_time.SetString(num_str.c_str(),
cast_set<uint>(num_str.length()),
+ root.GetAllocator());
+ root.AddMember("exec compaction time us", exec_compaction_time,
root.GetAllocator());
+ rapidjson::Value local_read_time;
+ num_str = std::to_string(local_read_time_us.load());
+ local_read_time.SetString(num_str.c_str(),
cast_set<uint>(num_str.length()),
+ root.GetAllocator());
+ root.AddMember("compaction local read time us", local_read_time,
root.GetAllocator());
+ rapidjson::Value remote_read_time;
+ num_str = std::to_string(remote_read_time_us.load());
+ remote_read_time.SetString(num_str.c_str(),
cast_set<uint>(num_str.length()),
+ root.GetAllocator());
+ root.AddMember("compaction remote read time us", remote_read_time,
root.GetAllocator());
// print all rowsets' version as an array
rapidjson::Document versions_arr;
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index 3474857b36a..b9cfdea3eb6 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -260,6 +260,10 @@ public:
int64_t last_cumu_no_suitable_version_ms = 0;
int64_t last_access_time_ms = 0;
+ std::atomic<int64_t> local_read_time_us = 0;
+ std::atomic<int64_t> remote_read_time_us = 0;
+ std::atomic<int64_t> exec_compaction_time_us = 0;
+
// Return merged extended schema
TabletSchemaSPtr merged_tablet_schema() const override;
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 026daa38c97..c9141e70132 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -38,6 +38,7 @@
#include "cloud/cloud_meta_mgr.h"
#include "cloud/cloud_storage_engine.h"
+#include "cloud/cloud_tablet.h"
#include "common/config.h"
#include "common/status.h"
#include "cpp/sync_point.h"
@@ -1439,6 +1440,12 @@ Status
CloudCompactionMixin::execute_compact_impl(int64_t permits) {
// 4. modify rowsets in memory
RETURN_IF_ERROR(modify_rowsets());
+ // update compaction status data
+ auto tablet = std::static_pointer_cast<CloudTablet>(_tablet);
+ tablet->local_read_time_us.fetch_add(_stats.cloud_local_read_time);
+ tablet->remote_read_time_us.fetch_add(_stats.cloud_remote_read_time);
+ tablet->exec_compaction_time_us.fetch_add(watch.get_elapse_time_us());
+
return Status::OK();
}
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 2797318c625..0467c011ab2 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -33,6 +33,7 @@
#include <utility>
#include <vector>
+#include "cloud/config.h"
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
@@ -147,6 +148,12 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet,
ReaderType reader_type,
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
stats_output->cached_bytes_total =
reader.stats().file_cache_stats.bytes_write_into_cache;
+ if (config::is_cloud_mode()) {
+ stats_output->cloud_local_read_time =
+ reader.stats().file_cache_stats.local_io_timer / 1000;
+ stats_output->cloud_remote_read_time =
+ reader.stats().file_cache_stats.remote_io_timer / 1000;
+ }
}
RETURN_NOT_OK_STATUS_WITH_WARN(dst_rowset_writer->flush(),
@@ -327,6 +334,12 @@ Status Merger::vertical_compact_one_group(
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
stats_output->cached_bytes_total =
reader.stats().file_cache_stats.bytes_write_into_cache;
+ if (config::is_cloud_mode()) {
+ stats_output->cloud_local_read_time =
+ reader.stats().file_cache_stats.local_io_timer / 1000;
+ stats_output->cloud_remote_read_time =
+ reader.stats().file_cache_stats.remote_io_timer / 1000;
+ }
}
RETURN_IF_ERROR(dst_rowset_writer->flush_columns(is_key));
diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h
index 76d053a7a79..302e606371a 100644
--- a/be/src/olap/merger.h
+++ b/be/src/olap/merger.h
@@ -43,6 +43,8 @@ class VerticalBlockReader;
class Merger {
public:
struct Statistics {
+ int64_t cloud_local_read_time = 0;
+ int64_t cloud_remote_read_time = 0;
// number of rows written to the destination rowset after merge
int64_t output_rows = 0;
int64_t merged_rows = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
