(doris) branch branch-2.0-var updated: [feature](invert index) add the inverted index to the file cache (#27678)

[kxiao]

This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch branch-2.0-var
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.0-var by this push:
     new db59b9046e5 [feature](invert index) add the inverted index to the file 
cache (#27678)
db59b9046e5 is described below

commit db59b9046e546c3a9ecbab72036765267d0e5d5b
Author: zzzxl <33418555+zzzxl1...@users.noreply.github.com>
AuthorDate: Wed Nov 29 21:36:21 2023 +0800

    [feature](invert index) add the inverted index to the file cache (#27678)
---
 .../rowset/segment_v2/inverted_index_compound_directory.cpp    | 10 ++++++++--
 .../olap/rowset/segment_v2/inverted_index_compound_directory.h |  3 +++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git 
a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp 
b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
index e563f61802f..de57eed6f85 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
@@ -263,7 +263,12 @@ bool DorisCompoundDirectory::FSIndexInput::open(const 
io::FileSystemSPtr& fs, co
     }
     SharedHandle* h = _CLNEW SharedHandle(path);
 
-    if (!fs->open_file(path, &h->_reader).ok()) {
+    io::FileDescription fd;
+    fd.path = path;
+    io::FileBlockCachePathPolicy cache_policy;
+    auto type = config::enable_file_cache ? config::file_cache_type : "";
+    io::FileReaderOptions reader_options(io::cache_type_from_string(type), 
cache_policy);
+    if (!fs->open_file(fd, reader_options, &h->_reader).ok()) {
         error.set(CL_ERR_IO, "open file error");
     }
 
@@ -301,6 +306,7 @@ DorisCompoundDirectory::FSIndexInput::FSIndexInput(const 
FSIndexInput& other)
     std::lock_guard<doris::Mutex> wlock(*other._handle->_shared_lock);
     _handle = _CL_POINTER(other._handle);
     _pos = other._handle->_fpos; //note where we are currently...
+    _io_ctx = other._io_ctx;
 }
 
 DorisCompoundDirectory::FSIndexInput::SharedHandle::SharedHandle(const char* 
path) {
@@ -367,7 +373,7 @@ void 
DorisCompoundDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_
 
     Slice result {b, (size_t)len};
     size_t bytes_read = 0;
-    if (!_handle->_reader->read_at(_pos, result, &bytes_read).ok()) {
+    if (!_handle->_reader->read_at(_pos, result, &bytes_read, &_io_ctx).ok()) {
         _CLTHROWA(CL_ERR_IO, "read past EOF");
     }
     bufferLength = len;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h 
b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
index b06f980f6bd..b89c6383539 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
@@ -29,6 +29,7 @@
 #include "CLucene/SharedHeader.h"
 #include "io/fs/file_reader_writer_fwd.h"
 #include "io/fs/file_system.h"
+#include "io/io_common.h"
 #include "util/lock.h"
 
 class CLuceneError;
@@ -136,10 +137,12 @@ class DorisCompoundDirectory::FSIndexInput : public 
lucene::store::BufferedIndex
 
     SharedHandle* _handle;
     int64_t _pos;
+    io::IOContext _io_ctx;
 
     FSIndexInput(SharedHandle* handle, int32_t buffer_size) : 
BufferedIndexInput(buffer_size) {
         this->_pos = 0;
         this->_handle = handle;
+        this->_io_ctx.reader_type = ReaderType::READER_QUERY;
     }
 
 protected:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org