This is an automated email from the ASF dual-hosted git repository.
ashingau pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d62e914205d [opt](profile) set datalake profile level as 1 (#25686)
d62e914205d is described below
commit d62e914205db5b77bc02f1419df86bb477004f23
Author: Ashin Gau <ashin...@users.noreply.github.com>
AuthorDate: Tue Oct 24 09:55:25 2023 +0800
[opt](profile) set datalake profile level as 1 (#25686)
Follow #25491, only the profile marked as 1 will be shown in simplified
profile.
---
be/src/io/cache/block/block_file_cache_profile.h | 33 +++++-----
be/src/io/fs/buffered_reader.h | 19 +++---
be/src/util/runtime_profile.h | 4 ++
be/src/vec/exec/format/orc/vorc_reader.cpp | 27 ++++----
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 76 +++++++++++-----------
5 files changed, 87 insertions(+), 72 deletions(-)
diff --git a/be/src/io/cache/block/block_file_cache_profile.h
b/be/src/io/cache/block/block_file_cache_profile.h
index 7635413cd97..fcc9ea503dd 100644
--- a/be/src/io/cache/block/block_file_cache_profile.h
+++ b/be/src/io/cache/block/block_file_cache_profile.h
@@ -116,22 +116,23 @@ struct FileCacheProfileReporter {
FileCacheProfileReporter(RuntimeProfile* profile) {
static const char* cache_profile = "FileCache";
- ADD_TIMER(profile, cache_profile);
- num_local_io_total =
- ADD_CHILD_COUNTER(profile, "NumLocalIOTotal", TUnit::UNIT,
cache_profile);
- num_remote_io_total =
- ADD_CHILD_COUNTER(profile, "NumRemoteIOTotal", TUnit::UNIT,
cache_profile);
- local_io_timer = ADD_CHILD_TIMER(profile, "LocalIOUseTimer",
cache_profile);
- remote_io_timer = ADD_CHILD_TIMER(profile, "RemoteIOUseTimer",
cache_profile);
- write_cache_io_timer = ADD_CHILD_TIMER(profile,
"WriteCacheIOUseTimer", cache_profile);
- bytes_write_into_cache =
- ADD_CHILD_COUNTER(profile, "BytesWriteIntoCache",
TUnit::BYTES, cache_profile);
- num_skip_cache_io_total =
- ADD_CHILD_COUNTER(profile, "NumSkipCacheIOTotal", TUnit::UNIT,
cache_profile);
- bytes_scanned_from_cache =
- ADD_CHILD_COUNTER(profile, "BytesScannedFromCache",
TUnit::BYTES, cache_profile);
- bytes_scanned_from_remote =
- ADD_CHILD_COUNTER(profile, "BytesScannedFromRemote",
TUnit::BYTES, cache_profile);
+ ADD_TIMER_WITH_LEVEL(profile, cache_profile, 1);
+ num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumLocalIOTotal", TUnit::UNIT,
+ cache_profile, 1);
+ num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumRemoteIOTotal", TUnit::UNIT,
+ cache_profile, 1);
+ local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile,
"LocalIOUseTimer", cache_profile, 1);
+ remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile,
"RemoteIOUseTimer", cache_profile, 1);
+ write_cache_io_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile, "WriteCacheIOUseTimer",
cache_profile, 1);
+ bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesWriteIntoCache",
+ TUnit::BYTES,
cache_profile, 1);
+ num_skip_cache_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"NumSkipCacheIOTotal",
+ TUnit::UNIT,
cache_profile, 1);
+ bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromCache",
+ TUnit::BYTES,
cache_profile, 1);
+ bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromRemote",
+ TUnit::BYTES,
cache_profile, 1);
}
void update(const FileCacheStatistics* statistics) {
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index f13a70c8743..586aa9546fe 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -143,14 +143,17 @@ public:
_remaining = TOTAL_BUFFER_SIZE;
if (_profile != nullptr) {
const char* random_profile = "MergedSmallIO";
- ADD_TIMER(_profile, random_profile);
- _copy_time = ADD_CHILD_TIMER(_profile, "CopyTime", random_profile);
- _read_time = ADD_CHILD_TIMER(_profile, "ReadTime", random_profile);
- _request_io = ADD_CHILD_COUNTER(_profile, "RequestIO",
TUnit::UNIT, random_profile);
- _merged_io = ADD_CHILD_COUNTER(_profile, "MergedIO", TUnit::UNIT,
random_profile);
- _request_bytes =
- ADD_CHILD_COUNTER(_profile, "RequestBytes", TUnit::BYTES,
random_profile);
- _read_bytes = ADD_CHILD_COUNTER(_profile, "MergedBytes",
TUnit::BYTES, random_profile);
+ ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1);
+ _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime",
random_profile, 1);
+ _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime",
random_profile, 1);
+ _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO",
TUnit::UNIT,
+ random_profile, 1);
+ _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO",
TUnit::UNIT,
+ random_profile, 1);
+ _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile,
"RequestBytes", TUnit::BYTES,
+ random_profile, 1);
+ _read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile,
"MergedBytes", TUnit::BYTES,
+ random_profile, 1);
}
}
diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h
index db36e6133d9..e4ddee2fe44 100644
--- a/be/src/util/runtime_profile.h
+++ b/be/src/util/runtime_profile.h
@@ -60,7 +60,11 @@ class TRuntimeProfileTree;
#define ADD_TIMER_WITH_LEVEL(profile, name, level) \
(profile)->add_counter_with_level(name, TUnit::TIME_NS, level)
#define ADD_CHILD_COUNTER(profile, name, type, parent)
(profile)->add_counter(name, type, parent)
+#define ADD_CHILD_COUNTER_WITH_LEVEL(profile, name, type, parent, level) \
+ (profile)->add_counter(name, type, parent, level)
#define ADD_CHILD_TIMER(profile, name, parent) (profile)->add_counter(name,
TUnit::TIME_NS, parent)
+#define ADD_CHILD_TIMER_WITH_LEVEL(profile, name, parent, level) \
+ (profile)->add_counter(name, TUnit::TIME_NS, parent, level)
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch>
MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
#define SCOPED_TIMER_ATOMIC(c) \
ScopedTimer<MonotonicStopWatch, std::atomic_bool>
MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp
b/be/src/vec/exec/format/orc/vorc_reader.cpp
index ca64f527653..21c94bad8c4 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -198,20 +198,25 @@ int64_t OrcReader::size() const {
void OrcReader::_init_profile() {
if (_profile != nullptr) {
static const char* orc_profile = "OrcReader";
- ADD_TIMER(_profile, orc_profile);
- _orc_profile.read_time = ADD_TIMER(_profile, "FileReadTime");
- _orc_profile.read_calls = ADD_COUNTER(_profile, "FileReadCalls",
TUnit::UNIT);
- _orc_profile.read_bytes = ADD_COUNTER(_profile, "FileReadBytes",
TUnit::BYTES);
- _orc_profile.column_read_time = ADD_CHILD_TIMER(_profile,
"ColumnReadTime", orc_profile);
- _orc_profile.get_batch_time = ADD_CHILD_TIMER(_profile,
"GetBatchTime", orc_profile);
+ ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1);
+ _orc_profile.read_time = ADD_TIMER_WITH_LEVEL(_profile,
"FileReadTime", 1);
+ _orc_profile.read_calls = ADD_COUNTER_WITH_LEVEL(_profile,
"FileReadCalls", TUnit::UNIT, 1);
+ _orc_profile.read_bytes =
+ ADD_COUNTER_WITH_LEVEL(_profile, "FileReadBytes",
TUnit::BYTES, 1);
+ _orc_profile.column_read_time =
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ColumnReadTime",
orc_profile, 1);
+ _orc_profile.get_batch_time =
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "GetBatchTime",
orc_profile, 1);
_orc_profile.create_reader_time =
- ADD_CHILD_TIMER(_profile, "CreateReaderTime", orc_profile);
- _orc_profile.init_column_time = ADD_CHILD_TIMER(_profile,
"InitColumnTime", orc_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CreateReaderTime",
orc_profile, 1);
+ _orc_profile.init_column_time =
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "InitColumnTime",
orc_profile, 1);
_orc_profile.set_fill_column_time =
- ADD_CHILD_TIMER(_profile, "SetFillColumnTime", orc_profile);
- _orc_profile.decode_value_time = ADD_CHILD_TIMER(_profile,
"DecodeValueTime", orc_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SetFillColumnTime",
orc_profile, 1);
+ _orc_profile.decode_value_time =
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeValueTime",
orc_profile, 1);
_orc_profile.decode_null_map_time =
- ADD_CHILD_TIMER(_profile, "DecodeNullMapTime", orc_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeNullMapTime",
orc_profile, 1);
}
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index ad97e519f87..25c1f46aed6 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -106,58 +106,60 @@ ParquetReader::~ParquetReader() {
void ParquetReader::_init_profile() {
if (_profile != nullptr) {
static const char* parquet_profile = "ParquetReader";
- ADD_TIMER(_profile, parquet_profile);
-
- _parquet_profile.filtered_row_groups =
- ADD_CHILD_COUNTER(_profile, "FilteredGroups", TUnit::UNIT,
parquet_profile);
- _parquet_profile.to_read_row_groups =
- ADD_CHILD_COUNTER(_profile, "ReadGroups", TUnit::UNIT,
parquet_profile);
- _parquet_profile.filtered_group_rows =
- ADD_CHILD_COUNTER(_profile, "FilteredRowsByGroup",
TUnit::UNIT, parquet_profile);
- _parquet_profile.filtered_page_rows =
- ADD_CHILD_COUNTER(_profile, "FilteredRowsByPage", TUnit::UNIT,
parquet_profile);
- _parquet_profile.lazy_read_filtered_rows =
- ADD_CHILD_COUNTER(_profile, "FilteredRowsByLazyRead",
TUnit::UNIT, parquet_profile);
- _parquet_profile.filtered_bytes =
- ADD_CHILD_COUNTER(_profile, "FilteredBytes", TUnit::BYTES,
parquet_profile);
- _parquet_profile.raw_rows_read =
- ADD_CHILD_COUNTER(_profile, "RawRowsRead", TUnit::UNIT,
parquet_profile);
- _parquet_profile.to_read_bytes =
- ADD_CHILD_COUNTER(_profile, "ReadBytes", TUnit::BYTES,
parquet_profile);
+ ADD_TIMER_WITH_LEVEL(_profile, parquet_profile, 1);
+
+ _parquet_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "FilteredGroups", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "ReadGroups", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "FilteredRowsByGroup", TUnit::UNIT, parquet_profile,
1);
+ _parquet_profile.filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "FilteredRowsByPage", TUnit::UNIT, parquet_profile,
1);
+ _parquet_profile.lazy_read_filtered_rows =
ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "FilteredRowsByLazyRead", TUnit::UNIT,
parquet_profile, 1);
+ _parquet_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1);
+ _parquet_profile.raw_rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.to_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "ReadBytes", TUnit::BYTES, parquet_profile, 1);
_parquet_profile.column_read_time =
- ADD_CHILD_TIMER(_profile, "ColumnReadTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ColumnReadTime",
parquet_profile, 1);
_parquet_profile.parse_meta_time =
- ADD_CHILD_TIMER(_profile, "ParseMetaTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ParseMetaTime",
parquet_profile, 1);
_parquet_profile.parse_footer_time =
- ADD_CHILD_TIMER(_profile, "ParseFooterTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ParseFooterTime",
parquet_profile, 1);
_parquet_profile.open_file_time =
- ADD_CHILD_TIMER(_profile, "FileOpenTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileOpenTime",
parquet_profile, 1);
_parquet_profile.open_file_num =
- ADD_CHILD_COUNTER(_profile, "FileNum", TUnit::UNIT,
parquet_profile);
+ ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT,
parquet_profile, 1);
_parquet_profile.page_index_filter_time =
- ADD_CHILD_TIMER(_profile, "PageIndexFilterTime",
parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PageIndexFilterTime",
parquet_profile, 1);
_parquet_profile.row_group_filter_time =
- ADD_CHILD_TIMER(_profile, "RowGroupFilterTime",
parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RowGroupFilterTime",
parquet_profile, 1);
- _parquet_profile.file_read_time = ADD_TIMER(_profile, "FileReadTime");
- _parquet_profile.file_read_calls = ADD_COUNTER(_profile,
"FileReadCalls", TUnit::UNIT);
+ _parquet_profile.file_read_time = ADD_TIMER_WITH_LEVEL(_profile,
"FileReadTime", 1);
+ _parquet_profile.file_read_calls =
+ ADD_COUNTER_WITH_LEVEL(_profile, "FileReadCalls", TUnit::UNIT,
1);
_parquet_profile.file_meta_read_calls =
- ADD_COUNTER(_profile, "FileMetaReadCalls", TUnit::UNIT);
- _parquet_profile.file_read_bytes = ADD_COUNTER(_profile,
"FileReadBytes", TUnit::BYTES);
+ ADD_COUNTER_WITH_LEVEL(_profile, "FileMetaReadCalls",
TUnit::UNIT, 1);
+ _parquet_profile.file_read_bytes =
+ ADD_COUNTER_WITH_LEVEL(_profile, "FileReadBytes",
TUnit::BYTES, 1);
_parquet_profile.decompress_time =
- ADD_CHILD_TIMER(_profile, "DecompressTime", parquet_profile);
- _parquet_profile.decompress_cnt =
- ADD_CHILD_COUNTER(_profile, "DecompressCount", TUnit::UNIT,
parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecompressTime",
parquet_profile, 1);
+ _parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.decode_header_time =
- ADD_CHILD_TIMER(_profile, "DecodeHeaderTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeHeaderTime",
parquet_profile, 1);
_parquet_profile.decode_value_time =
- ADD_CHILD_TIMER(_profile, "DecodeValueTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeValueTime",
parquet_profile, 1);
_parquet_profile.decode_dict_time =
- ADD_CHILD_TIMER(_profile, "DecodeDictTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeDictTime",
parquet_profile, 1);
_parquet_profile.decode_level_time =
- ADD_CHILD_TIMER(_profile, "DecodeLevelTime", parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeLevelTime",
parquet_profile, 1);
_parquet_profile.decode_null_map_time =
- ADD_CHILD_TIMER(_profile, "DecodeNullMapTime",
parquet_profile);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeNullMapTime",
parquet_profile, 1);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
