This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new e2e947e8892 [opt](index) add more inverted index query profile (#36545)
e2e947e8892 is described below
commit e2e947e8892b3998eb131f320ba1a7de197bde6d
Author: Kang <kxiao.ti...@gmail.com>
AuthorDate: Thu Jun 20 08:44:15 2024 +0800
[opt](index) add more inverted index query profile (#36545)
add more inverted index query profile:
- InvertedIndexQueryNullBitmapTime
- InvertedIndexQueryFileExistsTime
- InvertedIndexSearcherCacheHit
- InvertedIndexSearcherCacheMiss
---
be/src/olap/olap_common.h | 4 ++++
.../olap/rowset/segment_v2/inverted_index_cache.cpp | 8 +++++++-
.../olap/rowset/segment_v2/inverted_index_reader.cpp | 20 ++++++++++++++------
.../olap/rowset/segment_v2/inverted_index_reader.h | 8 +++++---
be/src/vec/exec/scan/new_olap_scan_node.cpp | 8 ++++++++
be/src/vec/exec/scan/new_olap_scan_node.h | 4 ++++
be/src/vec/exec/scan/new_olap_scanner.cpp | 4 ++++
7 files changed, 46 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index a8696b5c3b5..e5f029d2a29 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -357,10 +357,14 @@ struct OlapReaderStatistics {
int64_t inverted_index_query_timer = 0;
int64_t inverted_index_query_cache_hit = 0;
int64_t inverted_index_query_cache_miss = 0;
+ int64_t inverted_index_query_file_exists_timer = 0;
+ int64_t inverted_index_query_null_bitmap_timer = 0;
int64_t inverted_index_query_bitmap_copy_timer = 0;
int64_t inverted_index_query_bitmap_op_timer = 0;
int64_t inverted_index_searcher_open_timer = 0;
int64_t inverted_index_searcher_search_timer = 0;
+ int64_t inverted_index_searcher_cache_hit = 0;
+ int64_t inverted_index_searcher_cache_miss = 0;
int64_t output_index_result_column_timer = 0;
// number of segment filtered by column stat when creating seg iterator
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
index ff25178cc1a..035e28efabd 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
@@ -123,8 +123,11 @@ Status
InvertedIndexSearcherCache::get_index_searcher(const io::FileSystemSPtr&
InvertedIndexSearcherCache::CacheKey cache_key(file_path);
if (_lookup(cache_key, cache_handle)) {
+ stats->inverted_index_searcher_cache_hit++;
cache_handle->owned = false;
return Status::OK();
+ } else {
+ stats->inverted_index_searcher_cache_miss++;
}
cache_handle->owned = !use_cache;
@@ -134,7 +137,10 @@ Status
InvertedIndexSearcherCache::get_index_searcher(const io::FileSystemSPtr&
#ifndef BE_TEST
{
bool exists = false;
- RETURN_IF_ERROR(fs->exists(file_path, &exists));
+ {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_file_exists_timer);
+ RETURN_IF_ERROR(fs->exists(file_path, &exists));
+ }
if (!exists) {
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
"inverted index path: {} not exist.", file_path);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 79ab97ee50a..7db21a96c65 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -100,6 +100,7 @@ bool
InvertedIndexReader::_is_match_query(InvertedIndexQueryType query_type) {
}
bool InvertedIndexReader::indexExists(io::Path& index_file_path) {
+ // SCOPED_RAW_TIMER(&stats->inverted_index_query_file_exists_timer);
bool exists = false;
RETURN_IF_ERROR(_fs->exists(index_file_path, &exists));
return exists;
@@ -189,8 +190,10 @@ Status
InvertedIndexReader::get_index_search(OlapReaderStatistics* stats,
return Status::OK();
}
-Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle*
cache_handle,
+Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats,
+ InvertedIndexQueryCacheHandle*
cache_handle,
lucene::store::Directory* dir) {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_null_bitmap_timer);
lucene::store::IndexInput* null_bitmap_in = nullptr;
bool owned_dir = false;
try {
@@ -208,7 +211,10 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
}
bool exists = false;
- RETURN_IF_ERROR(_fs->exists(index_file_path, &exists));
+ {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_file_exists_timer);
+ RETURN_IF_ERROR(_fs->exists(index_file_path, &exists));
+ }
if (!exists) {
LOG(WARNING) << "inverted index: " << index_file_path.native() <<
" not exist.";
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
@@ -494,7 +500,7 @@ Status FullTextIndexReader::normal_index_search(
const IndexSearcherPtr& index_searcher, bool& null_bitmap_already_read,
const std::unique_ptr<lucene::search::Query>& query,
const std::shared_ptr<roaring::Roaring>& term_match_bitmap) {
- check_null_bitmap(index_searcher, null_bitmap_already_read);
+ check_null_bitmap(stats, index_searcher, null_bitmap_already_read);
try {
SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
@@ -578,13 +584,15 @@ Status FullTextIndexReader::match_regexp_index_search(
return Status::OK();
}
-void FullTextIndexReader::check_null_bitmap(const IndexSearcherPtr&
index_searcher,
+void FullTextIndexReader::check_null_bitmap(OlapReaderStatistics* stats,
+ const IndexSearcherPtr&
index_searcher,
bool& null_bitmap_already_read) {
// try to reuse index_searcher's directory to read null_bitmap to cache
// to avoid open directory additionally for null_bitmap
if (!null_bitmap_already_read) {
InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
- read_null_bitmap(&null_bitmap_cache_handle,
index_searcher->getReader()->directory());
+ read_null_bitmap(stats, &null_bitmap_cache_handle,
+ index_searcher->getReader()->directory());
null_bitmap_already_read = true;
}
}
@@ -699,7 +707,7 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
// try to reuse index_searcher's directory to read null_bitmap to cache
// to avoid open directory additionally for null_bitmap
InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
- read_null_bitmap(&null_bitmap_cache_handle,
index_searcher->getReader()->directory());
+ read_null_bitmap(stats, &null_bitmap_cache_handle,
index_searcher->getReader()->directory());
try {
if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 905fd12dfb3..83825642869 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -85,7 +85,8 @@ public:
const void* query_value, InvertedIndexQueryType
query_type,
uint32_t* count) = 0;
- Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
+ Status read_null_bitmap(OlapReaderStatistics* stats,
+ InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr);
virtual InvertedIndexReaderType type() = 0;
@@ -164,7 +165,8 @@ private:
const IndexSearcherPtr& index_searcher,
const std::shared_ptr<roaring::Roaring>&
term_match_bitmap);
- void check_null_bitmap(const IndexSearcherPtr& index_searcher, bool&
null_bitmap_already_read);
+ void check_null_bitmap(OlapReaderStatistics* stats, const
IndexSearcherPtr& index_searcher,
+ bool& null_bitmap_already_read);
Status match_phrase_prefix_index_search(
OlapReaderStatistics* stats, RuntimeState* runtime_state, const
std::wstring& field_ws,
@@ -307,7 +309,7 @@ public:
Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr) {
- return _reader->read_null_bitmap(cache_handle, dir);
+ return _reader->read_null_bitmap(_stats, cache_handle, dir);
}
[[nodiscard]] InvertedIndexReaderType get_inverted_index_reader_type()
const;
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp
b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index 4dc56c3f44c..3e55823f0ec 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -171,6 +171,10 @@ Status NewOlapScanNode::_init_profile() {
_inverted_index_query_cache_miss_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexQueryCacheMiss",
TUnit::UNIT);
_inverted_index_query_timer = ADD_TIMER(_segment_profile,
"InvertedIndexQueryTime");
+ _inverted_index_query_null_bitmap_timer =
+ ADD_TIMER(_segment_profile, "InvertedIndexQueryNullBitmapTime");
+ _inverted_index_query_file_exists_timer =
+ ADD_TIMER(_segment_profile, "InvertedIndexQueryFileExistsTime");
_inverted_index_query_bitmap_copy_timer =
ADD_TIMER(_segment_profile, "InvertedIndexQueryBitmapCopyTime");
_inverted_index_query_bitmap_op_timer =
@@ -179,6 +183,10 @@ Status NewOlapScanNode::_init_profile() {
ADD_TIMER(_segment_profile, "InvertedIndexSearcherOpenTime");
_inverted_index_searcher_search_timer =
ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchTime");
+ _inverted_index_searcher_cache_hit_counter =
+ ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheHit",
TUnit::UNIT);
+ _inverted_index_searcher_cache_miss_counter =
+ ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheMiss",
TUnit::UNIT);
_output_index_result_column_timer = ADD_TIMER(_segment_profile,
"OutputIndexResultColumnTimer");
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h
b/be/src/vec/exec/scan/new_olap_scan_node.h
index 7e48498e555..35b4290ecfa 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.h
+++ b/be/src/vec/exec/scan/new_olap_scan_node.h
@@ -191,10 +191,14 @@ private:
RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter =
nullptr;
RuntimeProfile::Counter* _inverted_index_query_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_query_file_exists_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_bitmap_op_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_search_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_searcher_cache_hit_counter =
nullptr;
+ RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter =
nullptr;
RuntimeProfile::Counter* _output_index_result_column_timer = nullptr;
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index 6d9a7964dff..9e077bcb9ef 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -587,6 +587,10 @@ void NewOlapScanner::_update_counters_before_close() {
COUNTER_UPDATE(olap_parent->_inverted_index_query_cache_miss_counter,
stats.inverted_index_query_cache_miss);
COUNTER_UPDATE(olap_parent->_inverted_index_query_timer,
stats.inverted_index_query_timer);
+ COUNTER_UPDATE(olap_parent->_inverted_index_query_file_exists_timer,
+ stats.inverted_index_query_file_exists_timer);
+ COUNTER_UPDATE(olap_parent->_inverted_index_query_null_bitmap_timer,
+ stats.inverted_index_query_null_bitmap_timer);
COUNTER_UPDATE(olap_parent->_inverted_index_query_bitmap_copy_timer,
stats.inverted_index_query_bitmap_copy_timer);
COUNTER_UPDATE(olap_parent->_inverted_index_query_bitmap_op_timer,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
