xiaokang commented on code in PR #28872:
URL: https://github.com/apache/doris/pull/28872#discussion_r1435465828
##########
be/src/olap/rowset/beta_rowset.cpp:
##########
@@ -347,11 +351,52 @@ Status BetaRowset::upload_to(io::RemoteFileSystem*
dest_fs, const RowsetId& new_
index_meta->get_index_suffix());
dest_paths.push_back(remote_inverted_index_file);
local_paths.push_back(local_inverted_index_file);
+ idx_remote_paths.push_back(remote_inverted_index_file);
}
}
}
auto st = dest_fs->batch_upload(local_paths, dest_paths);
if (st.ok()) {
+ // Pre-write the metadata of the inverted index into the file cache
+ if (config::enable_inverted_index_cache_on_cooldown) {
+ if (dest_fs->type() == io::FileSystemType::S3 &&
config::enable_file_cache) {
+ auto start = std::chrono::steady_clock::now();
+ for (auto& path : idx_remote_paths) {
+ std::shared_ptr<io::FileReader> file_reader = nullptr;
+ if (!dest_fs->open_file(path, &file_reader).ok()) {
+ continue;
+ }
+
+ const auto& url = file_reader->path().native();
+ size_t pos = url.rfind('/');
+ if (pos != std::string::npos) {
+ auto idx_path = url.substr(0, pos);
+ auto idx_name = url.substr(pos + 1);
+
+ try {
+ auto* directory = new DorisCompoundReader(
+
DorisCompoundDirectory::getDirectory(dest_fs, idx_path.c_str()),
+ idx_name.c_str(),
config::inverted_index_read_buffer_size);
+
+ OptionalIndexSearcherPtr result;
+ FulltextIndexSearcherBuilder builder;
+ auto res = builder.build(directory, result);
+ if (!res.ok()) {
+ LOG(WARNING) << "opening index reader err: "
<< res;
+ }
+ } catch (CLuceneError& err) {
+ LOG(WARNING) << "opening index reader clucene err:
" << err.what();
+ } catch (...) {
+ LOG(WARNING) << "opening index reader other err";
+ }
+ }
+ }
+ auto duration =
+
std::chrono::duration<float>(std::chrono::steady_clock::now() - start);
+ LOG(INFO) << "cooldown upload open invert index duration: " <<
duration.count();
Review Comment:
DEBUG
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
