yiguolei commented on code in PR #32641:
URL: https://github.com/apache/doris/pull/32641#discussion_r1542643876
##########
be/src/runtime/workload_group/workload_group_manager.cpp:
##########
@@ -135,6 +137,135 @@ void
WorkloadGroupMgr::delete_workload_group_by_ids(std::set<uint64_t> used_wg_i
<< "ms, deleted group size:" << deleted_task_groups.size();
}
+struct WorkloadGroupMemInfo {
+ int64_t total_mem_used = 0;
+ int64_t weighted_mem_used = 0;
+ bool is_low_wartermark = false;
+ bool is_high_wartermark = false;
+ double mem_used_ratio = 0;
+};
+void WorkloadGroupMgr::refresh_wg_memory_info() {
+ std::shared_lock<std::shared_mutex> r_lock(_group_mutex);
+ // workload group id -> workload group queries
+ std::unordered_map<uint64_t, std::unordered_map<TUniqueId,
std::weak_ptr<QueryContext>>>
+ all_wg_queries;
+ for (auto& [wg_id, wg] : _workload_groups) {
+ all_wg_queries.insert({wg_id, wg->queries()});
+ }
+
+ int64_t all_queries_mem_used = 0;
+
+ // calculate total memory used of each workload group and total memory
used of all queries
+ std::unordered_map<uint64_t, WorkloadGroupMemInfo> wgs_mem_info;
+ for (auto& [wg_id, wg_queries] : all_wg_queries) {
+ int64_t wg_total_mem_used = 0;
+ for (const auto& [query_id, query_ctx_ptr] : wg_queries) {
+ if (auto query_ctx = query_ctx_ptr.lock()) {
+ wg_total_mem_used +=
+ query_ctx->is_cancelled() ? 0 :
query_ctx->query_mem_tracker->consumption();
+ }
+ }
+ all_queries_mem_used += wg_total_mem_used;
+ wgs_mem_info[wg_id] = {wg_total_mem_used};
+ }
+
+ auto proc_vm_rss = PerfCounters::get_vm_rss();
+ if (all_queries_mem_used <= 0) {
+ return;
+ }
+
+ auto process_mem_used = doris::MemInfo::proc_mem_no_allocator_cache();
+ auto sys_mem_available = doris::MemInfo::sys_mem_available();
+ std::string debug_msg = fmt::format(
+ "\nProcess Memory Summary: process_vm_rss: {}, process mem: {},
sys mem available: "
+ "{}, all quries mem: {}",
+ PrettyPrinter::print(proc_vm_rss, TUnit::BYTES),
+ PrettyPrinter::print(process_mem_used, TUnit::BYTES),
+ PrettyPrinter::print(sys_mem_available, TUnit::BYTES),
+ PrettyPrinter::print(all_queries_mem_used, TUnit::BYTES));
+ LOG_EVERY_N(INFO, 5) << debug_msg;
+
+ if (proc_vm_rss < all_queries_mem_used) {
+ all_queries_mem_used = proc_vm_rss;
+ }
+
+ // process memory used is actually bigger than all_queries_mem_used,
+ // because memory of page cache, allocator cache, segment cache etc. are
included
+ // in process_mem_used.
+ // we count these cache memories equally on workload groups.
+ double ratio = (double)proc_vm_rss / (double)all_queries_mem_used;
+
+ for (auto& wg : _workload_groups) {
+ auto wg_mem_limit = wg.second->memory_limit();
+ auto& wg_mem_info = wgs_mem_info[wg.first];
+ wg_mem_info.weighted_mem_used = wg_mem_info.total_mem_used * ratio;
+ wg_mem_info.mem_used_ratio = (double)wg_mem_info.weighted_mem_used /
wg_mem_limit;
+
+ wg.second->set_weighted_memory_used(wg_mem_info.total_mem_used, ratio);
+
+ auto spill_low_water_mark =
wg.second->spill_threshold_low_water_mark();
+ auto spill_high_water_mark =
wg.second->spill_threashold_high_water_mark();
+ wg_mem_info.is_high_wartermark = (wg_mem_info.weighted_mem_used >
+ ((double)wg_mem_limit *
spill_high_water_mark / 100));
+ wg_mem_info.is_low_wartermark = (wg_mem_info.weighted_mem_used >
+ ((double)wg_mem_limit *
spill_low_water_mark / 100));
+
+ // calculate query weighted memory limit of task group
+ const auto& wg_queries = all_wg_queries[wg.first];
+ auto wg_query_count = wg_queries.size();
Review Comment:
这里如果query count < 1 感觉可以直接continue了；
那么216 行直接就是wg_mem_limit ) / wg_query_count
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
