This is an automated email from the ASF dual-hosted git repository.
wangbo pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0391d0f163f [branch-3.0]Refactor workload group metric (#46640)
(#46919)
0391d0f163f is described below
commit 0391d0f163f340f0d3a11d2486e289754dd7b622
Author: wangbo <wan...@selectdb.com>
AuthorDate: Tue Jan 14 12:05:47 2025 +0800
[branch-3.0]Refactor workload group metric (#46640) (#46919)
pick #46640
---
.../workload_group/workload_group_metrics.cpp | 67 ++++++++++------------
.../workload_group/workload_group_metrics.h | 27 +++++----
be/src/util/system_metrics.cpp | 4 +-
3 files changed, 46 insertions(+), 52 deletions(-)
diff --git a/be/src/runtime/workload_group/workload_group_metrics.cpp
b/be/src/runtime/workload_group/workload_group_metrics.cpp
index f1bdd0d14c2..2a34427a490 100644
--- a/be/src/runtime/workload_group/workload_group_metrics.cpp
+++ b/be/src/runtime/workload_group/workload_group_metrics.cpp
@@ -26,6 +26,13 @@
namespace doris {
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_cpu_time_sec,
doris::MetricUnit::SECONDS);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(workload_group_mem_used_bytes,
doris::MetricUnit::BYTES);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_remote_scan_bytes,
doris::MetricUnit::BYTES);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_total_local_scan_bytes,
+ doris::MetricUnit::BYTES);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_local_scan_bytes,
doris::MetricUnit::BYTES);
+
#include "common/compile_check_begin.h"
WorkloadGroupMetrics::~WorkloadGroupMetrics() {
@@ -33,39 +40,27 @@ WorkloadGroupMetrics::~WorkloadGroupMetrics() {
}
WorkloadGroupMetrics::WorkloadGroupMetrics(WorkloadGroup* wg) {
+ std::string wg_id_prefix = "workload_group_" + std::to_string(wg->id());
_entity = DorisMetrics::instance()->metric_registry()->register_entity(
- "workload_group." + std::to_string(wg->id()), {{"workload_group",
wg->name()}});
-
- _cpu_time_metric = std::make_unique<doris::MetricPrototype>(
- doris::MetricType::COUNTER, doris::MetricUnit::SECONDS,
"workload_group_cpu_time_sec");
- _cpu_time_counter =
-
(IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(_cpu_time_metric.get()));
-
- _mem_used_bytes_metric = std::make_unique<doris::MetricPrototype>(
- doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
"workload_group_mem_used_bytes");
- _mem_used_bytes_counter =
(IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
- _mem_used_bytes_metric.get()));
-
- _local_scan_bytes_metric = std::make_unique<doris::MetricPrototype>(
- doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
- "workload_group_local_scan_bytes");
- _local_scan_bytes_counter =
(IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
- _local_scan_bytes_metric.get()));
-
- _remote_scan_bytes_metric = std::make_unique<doris::MetricPrototype>(
- doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
- "workload_group_remote_scan_bytes");
- _remote_scan_bytes_counter =
(IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
- _remote_scan_bytes_metric.get()));
+ wg_id_prefix, {{"workload_group", wg->name()}, {"id",
std::to_string(wg->id())}});
+
+ INT_COUNTER_METRIC_REGISTER(_entity, workload_group_cpu_time_sec);
+ INT_GAUGE_METRIC_REGISTER(_entity, workload_group_mem_used_bytes);
+ INT_COUNTER_METRIC_REGISTER(_entity, workload_group_remote_scan_bytes);
+ INT_COUNTER_METRIC_REGISTER(_entity,
workload_group_total_local_scan_bytes);
std::vector<DataDirInfo>& data_dir_list =
io::BeConfDataDirReader::be_config_data_dir_list;
for (const auto& data_dir : data_dir_list) {
- std::unique_ptr<doris::MetricPrototype> metric =
std::make_unique<doris::MetricPrototype>(
- doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
- "workload_group_local_scan_bytes_" + data_dir.metric_name);
- _local_scan_bytes_counter_map.insert(
- {data_dir.path,
(IntCounter*)(_entity->register_metric<IntCounter>(metric.get()))});
- _local_scan_bytes_metric_map.insert({data_dir.path,
std::move(metric)});
+ std::string data_dir_metric_name = wg_id_prefix + "_io_" +
data_dir.metric_name;
+ std::shared_ptr<MetricEntity> io_entity =
+ DorisMetrics::instance()->metric_registry()->register_entity(
+ data_dir_metric_name, {{"workload_group", wg->name()},
+ {"path", data_dir.metric_name},
+ {"id",
std::to_string(wg->id())}});
+ IntCounter* workload_group_local_scan_bytes = nullptr;
+ INT_COUNTER_METRIC_REGISTER(io_entity,
workload_group_local_scan_bytes);
+ _local_scan_bytes_counter_map.insert({data_dir.path,
workload_group_local_scan_bytes});
+ _io_entity_list.push_back(io_entity);
}
}
@@ -78,7 +73,7 @@ void WorkloadGroupMetrics::update_memory_used_bytes(int64_t
memory_used) {
}
void WorkloadGroupMetrics::update_local_scan_io_bytes(std::string path,
uint64_t delta_io_bytes) {
- _local_scan_bytes_counter->increment(delta_io_bytes);
+ workload_group_total_local_scan_bytes->increment(delta_io_bytes);
auto range = _local_scan_bytes_counter_map.equal_range(path);
for (auto it = range.first; it != range.second; ++it) {
it->second->increment((int64_t)delta_io_bytes);
@@ -86,7 +81,7 @@ void
WorkloadGroupMetrics::update_local_scan_io_bytes(std::string path, uint64_t
}
void WorkloadGroupMetrics::update_remote_scan_io_bytes(uint64_t
delta_io_bytes) {
- _remote_scan_bytes_counter->increment(delta_io_bytes);
+ workload_group_remote_scan_bytes->increment(delta_io_bytes);
}
void WorkloadGroupMetrics::refresh_metrics() {
@@ -95,21 +90,21 @@ void WorkloadGroupMetrics::refresh_metrics() {
// cpu
uint64_t _current_cpu_time_nanos = _cpu_time_nanos.load();
uint64_t _cpu_time_sec = _current_cpu_time_nanos / (1000L * 1000L * 1000L);
- _cpu_time_counter->set_value(_cpu_time_sec);
+ workload_group_cpu_time_sec->set_value(_cpu_time_sec);
_per_sec_cpu_time_nanos = (_current_cpu_time_nanos - _last_cpu_time_nanos)
/ interval_second;
_last_cpu_time_nanos = _current_cpu_time_nanos;
// memory
- _mem_used_bytes_counter->set_value(_memory_used);
+ workload_group_mem_used_bytes->set_value(_memory_used);
// local scan
- int64_t current_local_scan_bytes = _local_scan_bytes_counter->value();
+ int64_t current_local_scan_bytes =
workload_group_total_local_scan_bytes->value();
_per_sec_local_scan_bytes =
(current_local_scan_bytes - _last_local_scan_bytes) /
interval_second;
_last_local_scan_bytes = current_local_scan_bytes;
// remote scan
- int64_t current_remote_scan_bytes = _remote_scan_bytes_counter->value();
+ int64_t current_remote_scan_bytes =
workload_group_remote_scan_bytes->value();
_per_sec_remote_scan_bytes =
(current_remote_scan_bytes - _last_remote_scan_bytes) /
interval_second;
_last_remote_scan_bytes = current_remote_scan_bytes;
@@ -128,7 +123,7 @@ int64_t
WorkloadGroupMetrics::get_remote_scan_bytes_per_second() {
}
int64_t WorkloadGroupMetrics::get_memory_used() {
- return _mem_used_bytes_counter->value();
+ return workload_group_mem_used_bytes->value();
}
} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_group/workload_group_metrics.h
b/be/src/runtime/workload_group/workload_group_metrics.h
index 5e43ec825c7..67085d8374f 100644
--- a/be/src/runtime/workload_group/workload_group_metrics.h
+++ b/be/src/runtime/workload_group/workload_group_metrics.h
@@ -22,6 +22,7 @@
#include <memory>
#include <string>
#include <unordered_map>
+#include <vector>
namespace doris {
@@ -29,7 +30,12 @@ class WorkloadGroup;
template <typename T>
class AtomicCounter;
-using IntAtomicCounter = AtomicCounter<int64_t>;
+using IntCounter = AtomicCounter<int64_t>;
+
+template <typename T>
+class AtomicGauge;
+using IntGuage = AtomicGauge<int64_t>;
+
class MetricEntity;
struct MetricPrototype;
@@ -58,19 +64,11 @@ public:
int64_t get_memory_used();
private:
- std::unique_ptr<doris::MetricPrototype> _cpu_time_metric {nullptr};
- std::unique_ptr<doris::MetricPrototype> _mem_used_bytes_metric {nullptr};
- std::unique_ptr<doris::MetricPrototype> _local_scan_bytes_metric {nullptr};
- std::unique_ptr<doris::MetricPrototype> _remote_scan_bytes_metric
{nullptr};
- // NOTE: _local_scan_bytes_metric is sum of all disk's IO
- std::unordered_multimap<std::string,
std::unique_ptr<doris::MetricPrototype>>
- _local_scan_bytes_metric_map;
-
- IntAtomicCounter* _cpu_time_counter {nullptr}; // used for metric
- IntAtomicCounter* _mem_used_bytes_counter {nullptr}; // used for metric
- IntAtomicCounter* _local_scan_bytes_counter {nullptr}; // used for metric
- IntAtomicCounter* _remote_scan_bytes_counter {nullptr}; // used for metric
- std::unordered_multimap<std::string, IntAtomicCounter*>
+ IntCounter* workload_group_cpu_time_sec {nullptr}; // used for
metric
+ IntGuage* workload_group_mem_used_bytes {nullptr}; // used for
metric
+ IntCounter* workload_group_remote_scan_bytes {nullptr}; // used for
metric
+ IntCounter* workload_group_total_local_scan_bytes {nullptr}; // used for
metric
+ std::unordered_multimap<std::string, IntCounter*>
_local_scan_bytes_counter_map; // used for metric
std::atomic<uint64_t> _cpu_time_nanos {0};
@@ -86,6 +84,7 @@ private:
std::atomic<uint64_t> _memory_used {0};
std::shared_ptr<MetricEntity> _entity {nullptr};
+ std::vector<std::shared_ptr<MetricEntity>> _io_entity_list;
};
} // namespace doris
\ No newline at end of file
diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp
index cf50e5be344..49b517f8dbd 100644
--- a/be/src/util/system_metrics.cpp
+++ b/be/src/util/system_metrics.cpp
@@ -49,8 +49,8 @@ struct CpuNumberMetrics {
INT_COUNTER_METRIC_REGISTER(entity, avail_cpu_num);
}
- IntAtomicCounter* host_cpu_num {nullptr};
- IntAtomicCounter* avail_cpu_num {nullptr};
+ IntCounter* host_cpu_num {nullptr};
+ IntCounter* avail_cpu_num {nullptr};
MetricEntity* entity = nullptr;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
