This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6bbd9df2890 [fix](memory) Fix pipelinex submit task attach memory
tracker (#33732)
6bbd9df2890 is described below
commit 6bbd9df28906d11259aba336f9ddd49661a3aa1f
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Wed Apr 17 17:46:23 2024 +0800
[fix](memory) Fix pipelinex submit task attach memory tracker (#33732)
---
be/src/agent/task_worker_pool.cpp | 4 ++--
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp | 4 ++--
be/src/olap/task/engine_publish_version_task.cpp | 4 ++--
be/src/olap/task/engine_publish_version_task.h | 2 +-
be/src/runtime/fold_constant_executor.cpp | 2 +-
be/src/runtime/fragment_mgr.cpp | 6 +++++-
be/src/service/backend_service.cpp | 2 +-
be/src/service/internal_service.cpp | 2 +-
8 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 39d96bc2098..c491c996b32 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -180,7 +180,7 @@ void alter_tablet(StorageEngine& engine, const
TAgentTaskRequest& agent_task_req
new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id;
new_schema_hash = agent_task_req.alter_tablet_req_v2.new_schema_hash;
auto mem_tracker = MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ MemTrackerLimiter::Type::OTHER,
fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}",
std::to_string(agent_task_req.alter_tablet_req_v2.base_tablet_id),
std::to_string(agent_task_req.alter_tablet_req_v2.new_tablet_id),
@@ -249,7 +249,7 @@ void alter_cloud_tablet(CloudStorageEngine& engine, const
TAgentTaskRequest& age
if (status.ok()) {
new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id;
auto mem_tracker = MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ MemTrackerLimiter::Type::OTHER,
fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}",
std::to_string(agent_task_req.alter_tablet_req_v2.base_tablet_id),
std::to_string(agent_task_req.alter_tablet_req_v2.new_tablet_id),
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index 9c784b1ced7..d31e7bd64ff 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -42,7 +42,7 @@
CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
_cal_delete_bitmap_req(cal_delete_bitmap_req),
_error_tablet_ids(error_tablet_ids),
_succ_tablet_ids(succ_tablet_ids) {
- _mem_tracker =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ _mem_tracker =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
"CloudEngineCalcDeleteBitmapTask");
}
@@ -134,7 +134,7 @@
CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask(
_transaction_id(transaction_id),
_version(version) {
_mem_tracker = MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ MemTrackerLimiter::Type::OTHER,
fmt::format("CloudTabletCalcDeleteBitmapTask#_transaction_id={}",
_transaction_id));
}
diff --git a/be/src/olap/task/engine_publish_version_task.cpp
b/be/src/olap/task/engine_publish_version_task.cpp
index 8a7ae8764d5..41d28a6124b 100644
--- a/be/src/olap/task/engine_publish_version_task.cpp
+++ b/be/src/olap/task/engine_publish_version_task.cpp
@@ -82,7 +82,7 @@ EnginePublishVersionTask::EnginePublishVersionTask(
_succ_tablets(succ_tablets),
_discontinuous_version_tablets(discontinuous_version_tablets),
_table_id_to_num_delta_rows(table_id_to_num_delta_rows) {
- _mem_tracker =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ _mem_tracker =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
"TabletPublishTxnTask");
}
@@ -357,7 +357,7 @@ TabletPublishTxnTask::TabletPublishTxnTask(StorageEngine&
engine,
_transaction_id(transaction_id),
_version(version),
_tablet_info(tablet_info),
-
_mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE,
+
_mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
"TabletPublishTxnTask")) {
_stats.submit_time_us = MonotonicMicros();
}
diff --git a/be/src/olap/task/engine_publish_version_task.h
b/be/src/olap/task/engine_publish_version_task.h
index 38e0cb1b6ac..e4824176368 100644
--- a/be/src/olap/task/engine_publish_version_task.h
+++ b/be/src/olap/task/engine_publish_version_task.h
@@ -121,7 +121,7 @@ public:
_partition_id(partition_id),
_transaction_id(transaction_id),
_version(version),
-
_mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE,
+
_mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
"AsyncTabletPublishTask")) {
_stats.submit_time_us = MonotonicMicros();
}
diff --git a/be/src/runtime/fold_constant_executor.cpp
b/be/src/runtime/fold_constant_executor.cpp
index 5ba384baf84..cf7695c86f2 100644
--- a/be/src/runtime/fold_constant_executor.cpp
+++ b/be/src/runtime/fold_constant_executor.cpp
@@ -160,7 +160,7 @@ Status FoldConstantExecutor::_init(const TQueryGlobals&
query_globals,
fragment_params.params = params;
fragment_params.protocol_version = PaloInternalServiceVersion::V1;
_mem_tracker = MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ MemTrackerLimiter::Type::OTHER,
fmt::format("FoldConstant:query_id={}", print_id(_query_id)));
_runtime_state =
RuntimeState::create_unique(fragment_params.params, query_options,
query_globals,
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index bc80dca583d..a377913ab98 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -198,6 +198,7 @@ std::string FragmentMgr::to_http_path(const std::string&
file_name) {
Status FragmentMgr::trigger_pipeline_context_report(
const ReportStatusRequest req,
std::shared_ptr<pipeline::PipelineFragmentContext>&& ctx) {
return _async_report_thread_pool->submit_func([this, req, ctx]() {
+ SCOPED_ATTACH_TASK(ctx->get_query_ctx()->query_mem_tracker);
coordinator_callback(req);
if (!req.done) {
ctx->refresh_next_report_time();
@@ -986,7 +987,10 @@ Status FragmentMgr::exec_plan_fragment(const
TPipelineFragmentParams& params,
for (size_t i = 0; i < target_size; i++) {
RETURN_IF_ERROR(_thread_pool->submit_func([&, i]() {
- prepare_status[i] = pre_and_submit(i);
+ {
+
SCOPED_ATTACH_TASK_WITH_ID(query_ctx->query_mem_tracker, params.query_id);
+ prepare_status[i] = pre_and_submit(i);
+ }
std::unique_lock<std::mutex> lock(m);
prepare_done++;
if (prepare_done == target_size) {
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index 8aab496ed2f..8dd48e2f083 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -105,7 +105,7 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
const auto& local_tablet_uid = local_tablet->tablet_uid();
std::shared_ptr<MemTrackerLimiter> mem_tracker =
MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
fmt::format("IngestBinlog#TxnId={}", txn_id));
+ MemTrackerLimiter::Type::OTHER,
fmt::format("IngestBinlog#TxnId={}", txn_id));
SCOPED_ATTACH_TASK(mem_tracker);
auto& request = arg->request;
diff --git a/be/src/service/internal_service.cpp
b/be/src/service/internal_service.cpp
index 2976928557e..fa4c427338c 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -732,7 +732,7 @@ void
PInternalService::fetch_table_schema(google::protobuf::RpcController* contr
const TFileScanRangeParams& params = file_scan_range.params;
std::shared_ptr<MemTrackerLimiter> mem_tracker =
MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ MemTrackerLimiter::Type::OTHER,
fmt::format("{}#{}", params.format_type, params.file_type));
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(mem_tracker);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
