This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 23cd49898c1 [Refact](inverted index) refact inverted index file writer
to make it more clear (#35459)
23cd49898c1 is described below
commit 23cd49898c1731b62c0ffcd33efa8d61d6283a8d
Author: airborne12 <airborn...@gmail.com>
AuthorDate: Wed Jun 19 19:35:53 2024 +0800
[Refact](inverted index) refact inverted index file writer to make it more
clear (#35459)
## Proposed changes
Issue Number: close #xxx
1. This PR refactors path handling and logging in the
InvertedIndexDescriptor and related components, ensuring clarity and
consistency across the codebase.
2. Updated initialization logic for DorisFSDirectory and
DorisRAMFSDirectory, removing confusing cfs and fs logic.
3. Removed DorisCompoundFileWriter and consolidated its logic into
InvertedIndexFileWriter.
3. Removed unused methods get_inverted_index_size and file_size from
several classes.
## Further comments
If this is a relatively large or complex change, kick off the discussion
at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why
you chose the solution you did and what alternatives you considered,
etc...
---
be/src/olap/compaction.cpp | 31 +-
be/src/olap/delta_writer.cpp | 39 +-
be/src/olap/rowset/beta_rowset.cpp | 220 +++++------
be/src/olap/rowset/beta_rowset_writer.cpp | 30 +-
be/src/olap/rowset/segcompaction.cpp | 19 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 36 --
be/src/olap/rowset/segment_v2/column_writer.h | 6 -
.../segment_v2/inverted_index_compound_reader.cpp | 4 -
.../olap/rowset/segment_v2/inverted_index_desc.cpp | 20 +-
.../olap/rowset/segment_v2/inverted_index_desc.h | 12 +-
.../segment_v2/inverted_index_file_reader.cpp | 34 +-
.../rowset/segment_v2/inverted_index_file_reader.h | 2 +-
.../segment_v2/inverted_index_file_writer.cpp | 406 +++++++++++----------
.../rowset/segment_v2/inverted_index_file_writer.h | 30 +-
.../segment_v2/inverted_index_fs_directory.cpp | 94 ++---
.../segment_v2/inverted_index_fs_directory.h | 30 +-
.../rowset/segment_v2/inverted_index_reader.cpp | 12 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 2 -
.../olap/rowset/segment_v2/inverted_index_writer.h | 2 -
be/src/olap/rowset/segment_v2/segment.cpp | 4 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 2 +-
be/src/olap/snapshot_manager.cpp | 5 +-
be/src/olap/task/index_builder.cpp | 28 +-
be/src/service/backend_service.cpp | 8 +-
be/src/service/internal_service.cpp | 28 +-
.../segment_v2/inverted_index_array_test.cpp | 4 +-
27 files changed, 516 insertions(+), 594 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 441a3fd140a..7b00ec1c09d 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -615,18 +615,13 @@ Status Compaction::do_inverted_index_compaction() {
auto seg_path = DORIS_TRY(rowset->segment_path(seg_id));
auto inverted_index_file_reader =
std::make_unique<InvertedIndexFileReader>(
- fs, std::string
{InvertedIndexDescriptor::get_index_path_prefix(seg_path)},
+ fs, std::string
{InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)},
_cur_tablet_schema->get_inverted_index_storage_format());
bool open_idx_file_cache = false;
- auto st =
inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
- open_idx_file_cache);
- if (!st.ok()) {
- LOG(ERROR) << "init inverted index "
- << InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_path))
- << " failed in compaction when init inverted index file
reader";
- return st;
- }
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+
inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
+ open_idx_file_cache),
+ "inverted_index_file_reader init failed");
inverted_index_file_readers[m.second] =
std::move(inverted_index_file_reader);
}
@@ -636,7 +631,7 @@ Status Compaction::do_inverted_index_compaction() {
dest_segment_num);
for (int i = 0; i < dest_segment_num; ++i) {
std::string index_path_prefix {
-
InvertedIndexDescriptor::get_index_path_prefix(ctx.segment_path(i))};
+
InvertedIndexDescriptor::get_index_file_path_prefix(ctx.segment_path(i))};
auto inverted_index_file_reader =
std::make_unique<InvertedIndexFileReader>(
ctx.fs(), index_path_prefix,
_cur_tablet_schema->get_inverted_index_storage_format());
@@ -649,10 +644,7 @@ Status Compaction::do_inverted_index_compaction() {
auto inverted_index_file_writer =
std::make_unique<InvertedIndexFileWriter>(
ctx.fs(), index_path_prefix, ctx.rowset_id.to_string(), i,
_cur_tablet_schema->get_inverted_index_storage_format());
- RETURN_NOT_OK_STATUS_WITH_WARN(
-
inverted_index_file_writer->initialize(index_not_need_to_compact),
- "failed to initialize inverted_index_file_writer for " +
- inverted_index_file_writer->get_index_file_path());
+
RETURN_IF_ERROR(inverted_index_file_writer->initialize(index_not_need_to_compact));
inverted_index_file_writers[i] =
std::move(inverted_index_file_writer);
} else if (st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>()) {
auto inverted_index_file_writer =
std::make_unique<InvertedIndexFileWriter>(
@@ -660,9 +652,8 @@ Status Compaction::do_inverted_index_compaction() {
_cur_tablet_schema->get_inverted_index_storage_format());
inverted_index_file_writers[i] =
std::move(inverted_index_file_writer);
} else {
- LOG(ERROR) << "init inverted index "
- <<
InvertedIndexDescriptor::get_index_path_v2(index_path_prefix)
- << " failed in compaction when create inverted index
file writer";
+ LOG(ERROR) << "inverted_index_file_reader init failed in index
compaction, error:"
+ << st;
return st;
}
}
@@ -797,7 +788,9 @@ void
Compaction::construct_skip_inverted_index(RowsetWriterContext& ctx) {
}
auto inverted_index_file_reader =
std::make_unique<InvertedIndexFileReader>(
- fs, std::string
{InvertedIndexDescriptor::get_index_path_prefix(*seg_path)},
+ fs,
+ std::string {
+
InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path)},
_cur_tablet_schema->get_inverted_index_storage_format());
bool open_idx_file_cache = false;
auto st =
inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index 7497df95dea..00c622df59f 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -278,12 +278,29 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(const
PNodeInfo& node_info)
local_segment_path(tablet_path,
cur_rowset->rowset_id().to_string(), segment_id);
int64_t segment_size = std::filesystem::file_size(seg_path);
request->mutable_segments_size()->insert({segment_id, segment_size});
- auto index_path_prefix =
InvertedIndexDescriptor::get_index_path_prefix(seg_path);
+ auto index_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path);
if (!indices_ids.empty()) {
- if (tablet_schema->get_inverted_index_storage_format() !=
+ if (tablet_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
+ for (auto index_meta : indices_ids) {
+ std::string inverted_index_file =
+ InvertedIndexDescriptor::get_index_file_path_v1(
+ index_path_prefix, index_meta.first,
index_meta.second);
+ int64_t size =
std::filesystem::file_size(inverted_index_file);
+ PTabletWriteSlaveRequest::IndexSize index_size;
+ index_size.set_indexid(index_meta.first);
+ index_size.set_size(size);
+ index_size.set_suffix_path(index_meta.second);
+ // Fetch the map value for the current segment_id.
+ // If it doesn't exist, this will insert a new
default-constructed IndexSizeMapValue
+ auto& index_size_map_value =
+
(*(request->mutable_inverted_indices_size()))[segment_id];
+ // Add the new index size to the map value.
+ *index_size_map_value.mutable_index_sizes()->Add() =
std::move(index_size);
+ }
+ } else {
std::string inverted_index_file =
-
InvertedIndexDescriptor::get_index_path_v2(index_path_prefix);
+
InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix);
int64_t size = std::filesystem::file_size(inverted_index_file);
PTabletWriteSlaveRequest::IndexSize index_size;
// special id for non-V1 format
@@ -296,22 +313,6 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(const
PNodeInfo& node_info)
(*(request->mutable_inverted_indices_size()))[segment_id];
// Add the new index size to the map value.
*index_size_map_value.mutable_index_sizes()->Add() =
std::move(index_size);
- } else {
- for (auto index_id : indices_ids) {
- std::string inverted_index_file =
InvertedIndexDescriptor::get_index_path_v1(
- seg_path, index_id.first, index_id.second);
- int64_t size =
std::filesystem::file_size(inverted_index_file);
- PTabletWriteSlaveRequest::IndexSize index_size;
- index_size.set_indexid(index_id.first);
- index_size.set_size(size);
- index_size.set_suffix_path(index_id.second);
- // Fetch the map value for the current segment_id.
- // If it doesn't exist, this will insert a new
default-constructed IndexSizeMapValue
- auto& index_size_map_value =
-
(*(request->mutable_inverted_indices_size()))[segment_id];
- // Add the new index size to the map value.
- *index_size_map_value.mutable_index_sizes()->Add() =
std::move(index_size);
- }
}
}
}
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index 16b18b6177b..a76cbe636ee 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -86,9 +86,10 @@ Status BetaRowset::get_inverted_index_size(size_t*
index_size) {
auto seg_path = DORIS_TRY(segment_path(seg_id));
int64_t file_size = 0;
- std::string inverted_index_file_path =
InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_path), index.index_id(),
- index.get_index_suffix());
+ std::string inverted_index_file_path =
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path),
+ index.index_id(), index.get_index_suffix());
RETURN_IF_ERROR(fs->file_size(inverted_index_file_path,
&file_size));
*index_size += file_size;
}
@@ -98,8 +99,8 @@ Status BetaRowset::get_inverted_index_size(size_t*
index_size) {
auto seg_path = DORIS_TRY(segment_path(seg_id));
int64_t file_size = 0;
- std::string inverted_index_file_path =
InvertedIndexDescriptor::get_index_path_v2(
- InvertedIndexDescriptor::get_index_path_prefix(seg_path));
+ std::string inverted_index_file_path =
InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path));
RETURN_IF_ERROR(fs->file_size(inverted_index_file_path,
&file_size));
*index_size += file_size;
}
@@ -114,14 +115,16 @@ void BetaRowset::clear_inverted_index_cache() {
continue;
}
- auto index_path_prefix =
InvertedIndexDescriptor::get_index_path_prefix(*seg_path);
+ auto index_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path);
for (const auto& column : tablet_schema()->columns()) {
const TabletIndex* index_meta =
tablet_schema()->get_inverted_index(*column);
if (index_meta) {
- std::string inverted_index_file =
InvertedIndexDescriptor::get_index_path_v1(
- index_path_prefix, index_meta->index_id(),
index_meta->get_index_suffix());
+ auto inverted_index_file_cache_key =
+ InvertedIndexDescriptor::get_index_file_cache_key(
+ index_path_prefix, index_meta->index_id(),
+ index_meta->get_index_suffix());
(void)segment_v2::InvertedIndexSearcherCache::instance()->erase(
- inverted_index_file);
+ inverted_index_file_cache_key);
}
}
}
@@ -214,25 +217,14 @@ Status BetaRowset::remove() {
success = false;
}
- if (_schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1 &&
- _schema->has_inverted_index()) {
- std::string inverted_index_file =
InvertedIndexDescriptor::get_index_path_v2(
- InvertedIndexDescriptor::get_index_path_prefix(seg_path));
- st = fs->delete_file(inverted_index_file);
- if (!st.ok()) {
- LOG(WARNING) << st.to_string();
- success = false;
- }
- }
-
- for (auto& column : _schema->columns()) {
- const TabletIndex* index_meta =
_schema->get_inverted_index(*column);
- if (index_meta) {
- std::string inverted_index_file =
InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_path),
- index_meta->index_id(),
index_meta->get_index_suffix());
- if (_schema->get_inverted_index_storage_format() ==
- InvertedIndexStorageFormatPB::V1) {
+ if (_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
+ for (auto& column : _schema->columns()) {
+ const TabletIndex* index_meta =
_schema->get_inverted_index(*column);
+ if (index_meta) {
+ std::string inverted_index_file =
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path),
+ index_meta->index_id(),
index_meta->get_index_suffix());
st = fs->delete_file(inverted_index_file);
if (!st.ok()) {
LOG(WARNING) << st.to_string();
@@ -240,6 +232,16 @@ Status BetaRowset::remove() {
}
}
}
+ } else {
+ if (_schema->has_inverted_index()) {
+ std::string inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path));
+ st = fs->delete_file(inverted_index_file);
+ if (!st.ok()) {
+ LOG(WARNING) << st.to_string();
+ success = false;
+ }
+ }
}
}
if (!success) {
@@ -300,48 +302,23 @@ Status BetaRowset::link_files_to(const std::string& dir,
RowsetId new_rowset_id,
status = Status::Error<OS_ERROR>("fault_inject link_file error");
return status;
});
- if (_schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1) {
- if (_schema->has_inverted_index() &&
- (without_index_uids == nullptr ||
without_index_uids->empty())) {
- std::string inverted_index_file_src =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(src_path));
- std::string inverted_index_file_dst =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(dst_path));
- bool index_dst_path_exist = false;
-
- if (!local_fs->exists(inverted_index_file_dst,
&index_dst_path_exist).ok() ||
- index_dst_path_exist) {
- status = Status::Error<FILE_ALREADY_EXIST>(
- "failed to create hard link, file already exist:
{}",
- inverted_index_file_dst);
- return status;
- }
- if (!local_fs->link_file(inverted_index_file_src,
inverted_index_file_dst).ok()) {
- status = Status::Error<OS_ERROR>(
- "fail to create hard link. from={}, to={},
errno={}",
- inverted_index_file_src, inverted_index_file_dst,
Errno::no());
- return status;
- }
- linked_success_files.push_back(inverted_index_file_dst);
- }
- } else {
+ if (_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
for (const auto& index : _schema->indexes()) {
if (index.index_type() != IndexType::INVERTED) {
continue;
}
-
auto index_id = index.index_id();
if (without_index_uids != nullptr &&
without_index_uids->count(index_id)) {
continue;
}
std::string inverted_index_src_file_path =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(src_path), index_id,
- index.get_index_suffix());
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(src_path),
+ index_id, index.get_index_suffix());
std::string inverted_index_dst_file_path =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(dst_path), index_id,
- index.get_index_suffix());
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(dst_path),
+ index_id, index.get_index_suffix());
bool index_file_exists = true;
RETURN_IF_ERROR(local_fs->exists(inverted_index_src_file_path,
&index_file_exists));
if (index_file_exists) {
@@ -370,6 +347,32 @@ Status BetaRowset::link_files_to(const std::string& dir,
RowsetId new_rowset_id,
<< inverted_index_src_file_path;
}
}
+ } else {
+ if (_schema->has_inverted_index() &&
+ (without_index_uids == nullptr ||
without_index_uids->empty())) {
+ std::string inverted_index_file_src =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(src_path));
+ std::string inverted_index_file_dst =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(dst_path));
+ bool index_dst_path_exist = false;
+
+ if (!local_fs->exists(inverted_index_file_dst,
&index_dst_path_exist).ok() ||
+ index_dst_path_exist) {
+ status = Status::Error<FILE_ALREADY_EXIST>(
+ "failed to create hard link, file already exist:
{}",
+ inverted_index_file_dst);
+ return status;
+ }
+ if (!local_fs->link_file(inverted_index_file_src,
inverted_index_file_dst).ok()) {
+ status = Status::Error<OS_ERROR>(
+ "fail to create hard link. from={}, to={},
errno={}",
+ inverted_index_file_src, inverted_index_file_dst,
Errno::no());
+ return status;
+ }
+ linked_success_files.push_back(inverted_index_file_dst);
+ }
}
}
return Status::OK();
@@ -391,29 +394,18 @@ Status BetaRowset::copy_files_to(const std::string& dir,
const RowsetId& new_row
}
auto src_path = local_segment_path(_tablet_path,
rowset_id().to_string(), i);
RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(src_path,
dst_path));
- if (_schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1) {
- if (_schema->has_inverted_index()) {
- std::string inverted_index_src_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(src_path));
- std::string inverted_index_dst_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(dst_path));
-
RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(inverted_index_src_file,
-
inverted_index_dst_file));
- LOG(INFO) << "success to copy file. from=" <<
inverted_index_src_file << ", "
- << "to=" << inverted_index_dst_file;
- }
- } else {
+ if (_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
// if (column.has_inverted_index()) {
const TabletIndex* index_meta =
_schema->get_inverted_index(*column);
if (index_meta) {
std::string inverted_index_src_file_path =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(src_path),
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(src_path),
index_meta->index_id(),
index_meta->get_index_suffix());
std::string inverted_index_dst_file_path =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(dst_path),
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(dst_path),
index_meta->index_id(),
index_meta->get_index_suffix());
RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(
inverted_index_src_file_path,
inverted_index_dst_file_path));
@@ -422,6 +414,19 @@ Status BetaRowset::copy_files_to(const std::string& dir,
const RowsetId& new_row
<< "to=" << inverted_index_dst_file_path;
}
}
+ } else {
+ if (_schema->has_inverted_index()) {
+ std::string inverted_index_src_file =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(src_path));
+ std::string inverted_index_dst_file =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(dst_path));
+
RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(inverted_index_src_file,
+
inverted_index_dst_file));
+ LOG(INFO) << "success to copy file. from=" <<
inverted_index_src_file << ", "
+ << "to=" << inverted_index_dst_file;
+ }
}
}
return Status::OK();
@@ -448,32 +453,38 @@ Status BetaRowset::upload_to(const StorageResource&
dest_fs, const RowsetId& new
auto local_seg_path = local_segment_path(_tablet_path,
rowset_id().to_string(), i);
dest_paths.emplace_back(remote_seg_path);
local_paths.emplace_back(local_seg_path);
- if (_schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1) {
- if (_schema->has_inverted_index()) {
- std::string remote_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(remote_seg_path));
- std::string local_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(local_seg_path));
- dest_paths.emplace_back(remote_inverted_index_file);
- local_paths.emplace_back(local_inverted_index_file);
- }
- } else {
+ if (_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
// if (column.has_inverted_index()) {
const TabletIndex* index_meta =
_schema->get_inverted_index(*column);
if (index_meta) {
std::string remote_inverted_index_file =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(remote_seg_path),
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ remote_seg_path),
index_meta->index_id(),
index_meta->get_index_suffix());
std::string local_inverted_index_file =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(local_seg_path),
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ local_seg_path),
index_meta->index_id(),
index_meta->get_index_suffix());
dest_paths.emplace_back(remote_inverted_index_file);
local_paths.emplace_back(local_inverted_index_file);
}
}
+ } else {
+ if (_schema->has_inverted_index()) {
+ std::string remote_inverted_index_file =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ remote_seg_path));
+ std::string local_inverted_index_file =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ local_seg_path));
+ dest_paths.emplace_back(remote_inverted_index_file);
+ local_paths.emplace_back(local_inverted_index_file);
+ }
}
}
auto st = dest_fs.fs->batch_upload(local_paths, dest_paths);
@@ -596,8 +607,8 @@ Status BetaRowset::add_to_binlog() {
continue;
}
auto index_id = index.index_id();
- auto index_file = InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_file), index_id,
+ auto index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_file), index_id,
index.get_index_suffix());
auto binlog_index_file = (std::filesystem::path(binlog_dir) /
std::filesystem::path(index_file).filename())
@@ -608,8 +619,8 @@ Status BetaRowset::add_to_binlog() {
}
} else {
if (_schema->has_inverted_index()) {
- auto index_file = InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_file));
+ auto index_file =
InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_file));
auto binlog_index_file = (std::filesystem::path(binlog_dir) /
std::filesystem::path(index_file).filename())
.string();
@@ -639,23 +650,26 @@ Status BetaRowset::calc_local_file_crc(uint32_t*
crc_value, int64_t* file_count)
for (int i = 0; i < num_segments(); ++i) {
auto local_seg_path = local_segment_path(_tablet_path,
rowset_id().to_string(), i);
local_paths.emplace_back(local_seg_path);
- if (_schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1) {
- if (_schema->has_inverted_index()) {
- std::string local_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(local_seg_path));
- local_paths.emplace_back(std::move(local_inverted_index_file));
- }
- } else {
+ if (_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
const TabletIndex* index_meta =
_schema->get_inverted_index(*column);
if (index_meta) {
std::string local_inverted_index_file =
- InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(local_seg_path),
+ InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ local_seg_path),
index_meta->index_id(),
index_meta->get_index_suffix());
local_paths.emplace_back(std::move(local_inverted_index_file));
}
}
+ } else {
+ if (_schema->has_inverted_index()) {
+ std::string local_inverted_index_file =
+ InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ local_seg_path));
+ local_paths.emplace_back(std::move(local_inverted_index_file));
+ }
}
}
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 2d16d9e46ee..d418a89a361 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -448,16 +448,18 @@ Status
BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u
_context.rowset_id.to_string(), seg_id)
:
BetaRowset::local_segment_path_segcompacted(
_context.tablet_path,
_context.rowset_id, begin, end);
- auto src_index_path_prefix =
InvertedIndexDescriptor::get_index_path_prefix(src_seg_path);
+ auto src_index_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(src_seg_path);
auto dst_seg_path = local_segment_path(_context.tablet_path,
_context.rowset_id.to_string(),
_num_segcompacted);
- auto dst_index_path_prefix =
InvertedIndexDescriptor::get_index_path_prefix(dst_seg_path);
+ auto dst_index_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(dst_seg_path);
- if (_context.tablet_schema->get_inverted_index_storage_format() !=
- InvertedIndexStorageFormatPB::V1) {
+ if (_context.tablet_schema->get_inverted_index_storage_format() >=
+ InvertedIndexStorageFormatPB::V2) {
if (_context.tablet_schema->has_inverted_index()) {
- auto src_idx_path =
InvertedIndexDescriptor::get_index_path_v2(src_index_path_prefix);
- auto dst_idx_path =
InvertedIndexDescriptor::get_index_path_v2(dst_index_path_prefix);
+ auto src_idx_path =
+
InvertedIndexDescriptor::get_index_file_path_v2(src_index_path_prefix);
+ auto dst_idx_path =
+
InvertedIndexDescriptor::get_index_file_path_v2(dst_index_path_prefix);
ret = rename(src_idx_path.c_str(), dst_idx_path.c_str());
if (ret) {
@@ -472,12 +474,12 @@ Status
BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u
if (_context.tablet_schema->has_inverted_index(*column)) {
const auto* index_info =
_context.tablet_schema->get_inverted_index(*column);
auto index_id = index_info->index_id();
- auto src_idx_path = InvertedIndexDescriptor::get_index_path_v1(
- src_index_path_prefix, index_id,
index_info->get_index_suffix());
- auto dst_idx_path = InvertedIndexDescriptor::get_index_path_v1(
- dst_index_path_prefix, index_id,
index_info->get_index_suffix());
if (_context.tablet_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
+ auto src_idx_path =
InvertedIndexDescriptor::get_index_file_path_v1(
+ src_index_path_prefix, index_id,
index_info->get_index_suffix());
+ auto dst_idx_path =
InvertedIndexDescriptor::get_index_file_path_v1(
+ dst_index_path_prefix, index_id,
index_info->get_index_suffix());
VLOG_DEBUG << "segcompaction skip this index. rename " <<
src_idx_path << " to "
<< dst_idx_path;
ret = rename(src_idx_path.c_str(), dst_idx_path.c_str());
@@ -488,8 +490,12 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t
begin, int64_t end, u
}
}
// Erase the origin index file cache
-
RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(src_idx_path));
-
RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(dst_idx_path));
+ auto src_idx_cache_key =
InvertedIndexDescriptor::get_index_file_cache_key(
+ src_index_path_prefix, index_id,
index_info->get_index_suffix());
+ auto dst_idx_cache_key =
InvertedIndexDescriptor::get_index_file_cache_key(
+ dst_index_path_prefix, index_id,
index_info->get_index_suffix());
+
RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(src_idx_cache_key));
+
RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(dst_idx_cache_key));
}
}
return Status::OK();
diff --git a/be/src/olap/rowset/segcompaction.cpp
b/be/src/olap/rowset/segcompaction.cpp
index 5b9b7c7a586..22a7049aa8f 100644
--- a/be/src/olap/rowset/segcompaction.cpp
+++ b/be/src/olap/rowset/segcompaction.cpp
@@ -133,9 +133,9 @@ Status
SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
RETURN_NOT_OK_STATUS_WITH_WARN(fs->delete_file(seg_path),
strings::Substitute("Failed to delete
file=$0", seg_path));
if (schema->has_inverted_index() &&
- schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1) {
- auto idx_path = InvertedIndexDescriptor::get_index_path_v2(
- InvertedIndexDescriptor::get_index_path_prefix(seg_path));
+ schema->get_inverted_index_storage_format() >=
InvertedIndexStorageFormatPB::V2) {
+ auto idx_path = InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path));
VLOG_DEBUG << "segcompaction index. delete file " << idx_path;
RETURN_NOT_OK_STATUS_WITH_WARN(
fs->delete_file(idx_path),
@@ -146,18 +146,21 @@ Status
SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
if (schema->has_inverted_index(*column)) {
const auto* index_info = schema->get_inverted_index(*column);
auto index_id = index_info->index_id();
- auto idx_path = InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_path), index_id,
- index_info->get_index_suffix());
- VLOG_DEBUG << "segcompaction index. delete file " << idx_path;
if (schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
+ auto idx_path =
InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), index_id,
+ index_info->get_index_suffix());
+ VLOG_DEBUG << "segcompaction index. delete file " <<
idx_path;
RETURN_NOT_OK_STATUS_WITH_WARN(
fs->delete_file(idx_path),
strings::Substitute("Failed to delete file=$0",
idx_path));
}
// Erase the origin index file cache
-
RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(idx_path));
+ auto idx_file_cache_key =
InvertedIndexDescriptor::get_index_file_cache_key(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), index_id,
+ index_info->get_index_suffix());
+
RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(idx_file_cache_key));
}
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp
b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 5b19f5669ac..016a932b276 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -465,7 +465,6 @@ Status ScalarColumnWriter::init() {
Status add_nulls(uint32_t count) override { return
Status::OK(); }
Status finish() override { return Status::OK(); }
int64_t size() const override { return 0; }
- int64_t file_size() const override { return 0; }
void close_on_error() override {}
};
@@ -635,14 +634,6 @@ Status ScalarColumnWriter::write_inverted_index() {
return Status::OK();
}
-size_t ScalarColumnWriter::get_inverted_index_size() {
- if (_opts.need_inverted_index) {
- auto size = _inverted_index_builder->file_size();
- return size == -1 ? 0 : size;
- }
- return 0;
-}
-
Status ScalarColumnWriter::write_bloom_filter_index() {
if (_opts.need_bloom_filter) {
return _bloom_filter_index_builder->finish(_file_writer,
_opts.meta->add_indexes());
@@ -798,17 +789,6 @@ Status StructColumnWriter::write_inverted_index() {
return Status::OK();
}
-size_t StructColumnWriter::get_inverted_index_size() {
- size_t total_size = 0;
- if (_opts.need_inverted_index) {
- for (auto& column_writer : _sub_column_writers) {
- auto size = column_writer->get_inverted_index_size();
- total_size += (size == -1 ? 0 : size);
- }
- }
- return total_size;
-}
-
Status StructColumnWriter::append_nullable(const uint8_t* null_map, const
uint8_t** ptr,
size_t num_rows) {
RETURN_IF_ERROR(append_data(ptr, num_rows));
@@ -921,14 +901,6 @@ Status ArrayColumnWriter::write_inverted_index() {
return Status::OK();
}
-size_t ArrayColumnWriter::get_inverted_index_size() {
- if (_opts.need_inverted_index) {
- auto size = _inverted_index_builder->file_size();
- return size == -1 ? 0 : size;
- }
- return 0;
-}
-
// batch append data for array
Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
// data_ptr contains
@@ -1169,12 +1141,4 @@ Status MapColumnWriter::write_inverted_index() {
return Status::OK();
}
-size_t MapColumnWriter::get_inverted_index_size() {
- if (_opts.need_inverted_index) {
- auto size = _inverted_index_builder->file_size();
- return size == -1 ? 0 : size;
- }
- return 0;
-}
-
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h
b/be/src/olap/rowset/segment_v2/column_writer.h
index 410ae3eb768..acda2b75fb2 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -156,8 +156,6 @@ public:
virtual Status write_inverted_index() = 0;
- virtual size_t get_inverted_index_size() = 0;
-
virtual Status write_bloom_filter_index() = 0;
virtual ordinal_t get_next_rowid() const = 0;
@@ -208,7 +206,6 @@ public:
Status write_zone_map() override;
Status write_bitmap_index() override;
Status write_inverted_index() override;
- size_t get_inverted_index_size() override;
Status write_bloom_filter_index() override;
ordinal_t get_next_rowid() const override { return _next_rowid; }
@@ -334,7 +331,6 @@ public:
return Status::OK();
}
Status write_inverted_index() override;
- size_t get_inverted_index_size() override;
Status write_bloom_filter_index() override {
if (_opts.need_bloom_filter) {
return Status::NotSupported("struct not support bloom filter
index");
@@ -386,7 +382,6 @@ public:
return Status::OK();
}
Status write_inverted_index() override;
- size_t get_inverted_index_size() override;
Status write_bloom_filter_index() override {
if (_opts.need_bloom_filter) {
return Status::NotSupported("array not support bloom filter
index");
@@ -425,7 +420,6 @@ public:
Status write_data() override;
Status write_ordinal_index() override;
Status write_inverted_index() override;
- size_t get_inverted_index_size() override;
Status append_nulls(size_t num_rows) override;
Status finish_current_page() override;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
index 5e6d8747a2d..67c3ac5253f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
@@ -228,10 +228,6 @@ lucene::store::Directory*
DorisCompoundReader::getDirectory() {
return dir;
}
-std::string DorisCompoundReader::getPath() const {
- return ((DorisFSDirectory*)dir)->getCfsDirName();
-}
-
int64_t DorisCompoundReader::fileModified(const char* name) const {
return dir->fileModified(name);
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
index bc25534eaaa..8eac73f13a6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
@@ -36,9 +36,9 @@ std::string
InvertedIndexDescriptor::get_temporary_index_path(std::string_view t
// InvertedIndexStorageFormat V1
// {prefix}_{index_id}@{suffix}.idx
-std::string InvertedIndexDescriptor::get_index_path_v1(std::string_view
index_path_prefix,
- int64_t index_id,
- std::string_view
index_path_suffix) {
+std::string InvertedIndexDescriptor::get_index_file_path_v1(std::string_view
index_path_prefix,
+ int64_t index_id,
+ std::string_view
index_path_suffix) {
std::string suffix =
index_path_suffix.empty() ? "" : std::string {"@"} +
index_path_suffix.data();
return fmt::format("{}_{}{}{}", index_path_prefix, index_id, suffix,
index_suffix);
@@ -46,7 +46,7 @@ std::string
InvertedIndexDescriptor::get_index_path_v1(std::string_view index_pa
// InvertedIndexStorageFormat V2
// {prefix}.idx
-std::string InvertedIndexDescriptor::get_index_path_v2(std::string_view
index_path_prefix) {
+std::string InvertedIndexDescriptor::get_index_file_path_v2(std::string_view
index_path_prefix) {
return fmt::format("{}{}", index_path_prefix, index_suffix);
}
@@ -54,10 +54,20 @@ std::string
InvertedIndexDescriptor::get_index_path_v2(std::string_view index_pa
//
{storage_dir}/data/{shard_id}/{tablet_id}/{schema_hash}/{rowset_id}_{seg_id}
// remote path v0 prefix:
// data/{tablet_id}/{rowset_id}_{seg_id}
-std::string_view
InvertedIndexDescriptor::get_index_path_prefix(std::string_view segment_path) {
+std::string_view InvertedIndexDescriptor::get_index_file_path_prefix(
+ std::string_view segment_path) {
CHECK(segment_path.ends_with(segment_suffix));
segment_path.remove_suffix(segment_suffix.size());
return segment_path;
}
+// {prefix}_{index_id}@{suffix} for inverted index cache
+std::string InvertedIndexDescriptor::get_index_file_cache_key(std::string_view
index_path_prefix,
+ int64_t index_id,
+ std::string_view
index_path_suffix) {
+ std::string suffix =
+ index_path_suffix.empty() ? "" : std::string {"@"} +
index_path_suffix.data();
+ return fmt::format("{}_{}{}", index_path_prefix, index_id, suffix);
+}
+
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
index 01f0636f943..37f9cf3f4a1 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
@@ -35,12 +35,16 @@ public:
int64_t index_id,
std::string_view
index_path_suffix);
// InvertedIndexStorageFormat V1
- static std::string get_index_path_v1(std::string_view index_path_prefix,
int64_t index_id,
- std::string_view index_path_suffix);
+ static std::string get_index_file_path_v1(std::string_view
index_path_prefix, int64_t index_id,
+ std::string_view
index_path_suffix);
// InvertedIndexStorageFormat V2
- static std::string get_index_path_v2(std::string_view index_path_prefix);
+ static std::string get_index_file_path_v2(std::string_view
index_path_prefix);
- static std::string_view get_index_path_prefix(std::string_view
segment_path);
+ static std::string_view get_index_file_path_prefix(std::string_view
segment_path);
+
+ static std::string get_index_file_cache_key(std::string_view
index_path_prefix,
+ int64_t index_id,
+ std::string_view
index_path_suffix);
static const char* get_temporary_null_bitmap_file_name() { return
"null_bitmap"; }
static const char* get_temporary_bkd_index_data_file_name() { return
"bkd"; }
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index 14d658d90b6..7a744ea939e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -37,7 +37,7 @@ Status InvertedIndexFileReader::init(int32_t
read_buffer_size, bool open_idx_fil
}
Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) {
- auto index_file_full_path =
InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix);
+ auto index_file_full_path =
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
std::unique_lock<std::shared_mutex> lock(_mutex); // Lock for writing
try {
@@ -145,10 +145,10 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::_open(
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
DorisFSDirectory* dir = nullptr;
- auto index_path =
InvertedIndexDescriptor::get_index_path_v1(_index_path_prefix, index_id,
-
index_suffix);
+ auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1(
+ _index_path_prefix, index_id, index_suffix);
try {
- std::filesystem::path path(index_path);
+ std::filesystem::path path(index_file_path);
dir = DorisFSDirectoryFactory::getDirectory(_fs,
path.parent_path().c_str());
compound_reader = std::make_unique<DorisCompoundReader>(
dir, path.filename().c_str(), _read_buffer_size,
_open_idx_file_cache);
@@ -158,7 +158,7 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::_open(
_CLDELETE(dir)
}
return
ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "CLuceneError occur when open idx file {}, error msg: {}",
index_path,
+ "CLuceneError occur when open idx file {}, error msg: {}",
index_file_path,
err.what()));
}
} else {
@@ -166,7 +166,7 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::_open(
if (_stream == nullptr) {
return
ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
"CLuceneError occur when open idx file {}, stream is
nullptr",
-
InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix)));
+
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)));
}
// Check if the specified index exists
@@ -176,7 +176,8 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::_open(
errMsg << "No index with id " << index_id << " found";
return
ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"CLuceneError occur when open idx file {}, error msg: {}",
-
InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix), errMsg.str()));
+
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix),
+ errMsg.str()));
}
// Need to clone resource here, because index searcher cache need it.
compound_reader = std::make_unique<DorisCompoundReader>(
@@ -191,29 +192,31 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::open(
return _open(index_id, index_suffix);
}
-std::string InvertedIndexFileReader::get_index_file_key(const TabletIndex*
index_meta) const {
- return InvertedIndexDescriptor::get_index_path_v1(_index_path_prefix,
index_meta->index_id(),
-
index_meta->get_index_suffix());
+std::string InvertedIndexFileReader::get_index_file_cache_key(const
TabletIndex* index_meta) const {
+ return InvertedIndexDescriptor::get_index_file_cache_key(
+ _index_path_prefix, index_meta->index_id(),
index_meta->get_index_suffix());
}
std::string InvertedIndexFileReader::get_index_file_path(const TabletIndex*
index_meta) const {
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
- return InvertedIndexDescriptor::get_index_path_v1(
+ return InvertedIndexDescriptor::get_index_file_path_v1(
_index_path_prefix, index_meta->index_id(),
index_meta->get_index_suffix());
}
- return InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix);
+ return InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
}
Status InvertedIndexFileReader::index_file_exist(const TabletIndex*
index_meta, bool* res) const {
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
- auto index_file_path = get_index_file_path(index_meta);
+ auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1(
+ _index_path_prefix, index_meta->index_id(),
index_meta->get_index_suffix());
return _fs->exists(index_file_path, res);
} else {
std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading
if (_stream == nullptr) {
*res = false;
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
- "idx file {} is not opened",
get_index_file_path(index_meta));
+ "idx file {} is not opened",
+
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix));
}
// Check if the specified index exists
auto index_it = _indices_entries.find(
@@ -235,7 +238,8 @@ Status InvertedIndexFileReader::has_null(const TabletIndex*
index_meta, bool* re
std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading
if (_stream == nullptr) {
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
- "idx file {} is not opened", get_index_file_path(index_meta));
+ "idx file {} is not opened",
+
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix));
}
// Check if the specified index exists
auto index_it = _indices_entries.find(
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
index cc5cd6cdb5d..fc0dd6e43d1 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
@@ -60,7 +60,7 @@ public:
bool open_idx_file_cache = false);
Result<std::unique_ptr<DorisCompoundReader>> open(const TabletIndex*
index_meta) const;
void debug_file_entries();
- std::string get_index_file_key(const TabletIndex* index_meta) const;
+ std::string get_index_file_cache_key(const TabletIndex* index_meta) const;
std::string get_index_file_path(const TabletIndex* index_meta) const;
Status index_file_exist(const TabletIndex* index_meta, bool* res) const;
Status has_null(const TabletIndex* index_meta, bool* res) const;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
index b7cb9591b88..cdd26fecf87 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
@@ -31,10 +31,6 @@
namespace doris::segment_v2 {
-std::string InvertedIndexFileWriter::get_index_file_path() const {
- return InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix);
-}
-
Status InvertedIndexFileWriter::initialize(InvertedIndexDirectoryMap&
indices_dirs) {
_indices_dirs = std::move(indices_dirs);
return Status::OK();
@@ -46,12 +42,6 @@ Result<DorisFSDirectory*>
InvertedIndexFileWriter::open(const TabletIndex* index
auto local_fs_index_path =
InvertedIndexDescriptor::get_temporary_index_path(
tmp_file_dir.native(), _rowset_id, _seg_id, index_meta->index_id(),
index_meta->get_index_suffix());
- auto index_path = InvertedIndexDescriptor::get_index_path_v1(
- _index_path_prefix, index_meta->index_id(),
index_meta->get_index_suffix());
- // FIXME(plat1ko): hardcode
- index_path =
- index_path.substr(0, index_path.size() -
InvertedIndexDescriptor::index_suffix.size());
-
bool exists = false;
auto st = local_fs->exists(local_fs_index_path, &exists);
if (!st.ok()) {
@@ -65,10 +55,8 @@ Result<DorisFSDirectory*>
InvertedIndexFileWriter::open(const TabletIndex* index
}
bool can_use_ram_dir = true;
- bool use_compound_file_writer = false;
auto* dir = DorisFSDirectoryFactory::getDirectory(local_fs,
local_fs_index_path.c_str(),
-
use_compound_file_writer, can_use_ram_dir,
- nullptr, _fs,
index_path.c_str());
+ can_use_ram_dir);
_indices_dirs.emplace(std::make_pair(index_meta->index_id(),
index_meta->get_index_suffix()),
std::unique_ptr<DorisFSDirectory>(dir));
return dir;
@@ -125,22 +113,24 @@ Status InvertedIndexFileWriter::close() {
if (_indices_dirs.empty()) {
return Status::OK();
}
- try {
- if (_storage_format == InvertedIndexStorageFormatPB::V1) {
+ if (_storage_format == InvertedIndexStorageFormatPB::V1) {
+ try {
+ _file_size = write_v1();
for (const auto& entry : _indices_dirs) {
const auto& dir = entry.second;
- auto* cfsWriter = _CLNEW DorisCompoundFileWriter(dir.get());
- // write compound file
- _file_size += cfsWriter->writeCompoundFile();
// delete index path, which contains separated inverted index
files
if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") ==
0) {
auto* compound_dir =
static_cast<DorisFSDirectory*>(dir.get());
compound_dir->deleteDirectory();
}
- _CLDELETE(cfsWriter)
}
- } else {
- _file_size = write();
+ } catch (CLuceneError& err) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError occur when close, error msg: {}",
err.what());
+ }
+ } else {
+ try {
+ _file_size = write_v2();
for (const auto& entry : _indices_dirs) {
const auto& dir = entry.second;
// delete index path, which contains separated inverted index
files
@@ -149,20 +139,206 @@ Status InvertedIndexFileWriter::close() {
compound_dir->deleteDirectory();
}
}
+ } catch (CLuceneError& err) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError occur when close idx file {}, error msg: {}",
+
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix),
+ err.what());
}
- } catch (CLuceneError& err) {
- return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "CLuceneError occur when close idx file {}, error msg: {}",
-
InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix), err.what());
}
return Status::OK();
}
-size_t InvertedIndexFileWriter::write() {
+void InvertedIndexFileWriter::sort_files(std::vector<FileInfo>& file_infos) {
+ auto file_priority = [](const std::string& filename) {
+ if (filename.find("segments") != std::string::npos) {
+ return 1;
+ }
+ if (filename.find("fnm") != std::string::npos) {
+ return 2;
+ }
+ if (filename.find("tii") != std::string::npos) {
+ return 3;
+ }
+ return 4; // Other files
+ };
+ std::sort(file_infos.begin(), file_infos.end(), [&](const FileInfo& a,
const FileInfo& b) {
+ int32_t priority_a = file_priority(a.filename);
+ int32_t priority_b = file_priority(b.filename);
+ if (priority_a != priority_b) {
+ return priority_a < priority_b;
+ }
+ return a.filesize < b.filesize;
+ });
+}
+
+void InvertedIndexFileWriter::copyFile(const char* fileName,
lucene::store::Directory* dir,
+ lucene::store::IndexOutput* output,
uint8_t* buffer,
+ int64_t bufferLength) {
+ lucene::store::IndexInput* tmp = nullptr;
+ CLuceneError err;
+ if (!dir->openInput(fileName, tmp, err)) {
+ throw err;
+ }
+
+ std::unique_ptr<lucene::store::IndexInput> input(tmp);
+ int64_t start_ptr = output->getFilePointer();
+ int64_t length = input->length();
+ int64_t remainder = length;
+ int64_t chunk = bufferLength;
+
+ while (remainder > 0) {
+ int64_t len = std::min(std::min(chunk, length), remainder);
+ input->readBytes(buffer, len);
+ output->writeBytes(buffer, len);
+ remainder -= len;
+ }
+ if (remainder != 0) {
+ std::ostringstream errMsg;
+ errMsg << "Non-zero remainder length after copying: " << remainder <<
" (id: " << fileName
+ << ", length: " << length << ", buffer size: " << chunk << ")";
+ err.set(CL_ERR_IO, errMsg.str().c_str());
+ throw err;
+ }
+
+ int64_t end_ptr = output->getFilePointer();
+ int64_t diff = end_ptr - start_ptr;
+ if (diff != length) {
+ std::ostringstream errMsg;
+ errMsg << "Difference in the output file offsets " << diff
+ << " does not match the original file length " << length;
+ err.set(CL_ERR_IO, errMsg.str().c_str());
+ throw err;
+ }
+ input->close();
+}
+
+size_t InvertedIndexFileWriter::write_v1() {
+ size_t total_size = 0;
+ for (const auto& entry : _indices_dirs) {
+ const int64_t index_id = entry.first.first;
+ const auto& index_suffix = entry.first.second;
+ try {
+ const auto& directory = entry.second;
+ std::vector<std::string> files;
+ directory->list(&files);
+ // remove write.lock file
+ auto it = std::find(files.begin(), files.end(),
DorisFSDirectory::WRITE_LOCK_FILE);
+ if (it != files.end()) {
+ files.erase(it);
+ }
+
+ std::vector<FileInfo> sorted_files;
+ for (auto file : files) {
+ FileInfo file_info;
+ file_info.filename = file;
+ file_info.filesize = directory->fileLength(file.c_str());
+ sorted_files.emplace_back(std::move(file_info));
+ }
+ sort_files(sorted_files);
+
+ int32_t file_count = sorted_files.size();
+
+ io::Path cfs_path(InvertedIndexDescriptor::get_index_file_path_v1(
+ _index_path_prefix, index_id, index_suffix));
+ auto idx_path = cfs_path.parent_path();
+ std::string idx_name = cfs_path.filename();
+ // write file entries to ram directory to get header length
+ lucene::store::RAMDirectory ram_dir;
+ auto* out_idx = ram_dir.createOutput(idx_name.c_str());
+ if (out_idx == nullptr) {
+ LOG(WARNING) << "Write compound file error: RAMDirectory
output is nullptr.";
+ _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error");
+ }
+
+ std::unique_ptr<lucene::store::IndexOutput> ram_output(out_idx);
+ ram_output->writeVInt(file_count);
+ // write file entries in ram directory
+ // number of files, which data are in header
+ int header_file_count = 0;
+ int64_t header_file_length = 0;
+ const int64_t buffer_length = 16384;
+ uint8_t ram_buffer[buffer_length];
+ for (auto file : sorted_files) {
+ ram_output->writeString(file.filename); // file name
+ ram_output->writeLong(0); // data offset
+ ram_output->writeLong(file.filesize); // file length
+ header_file_length += file.filesize;
+ if (header_file_length <=
DorisFSDirectory::MAX_HEADER_DATA_SIZE) {
+ copyFile(file.filename.c_str(), directory.get(),
ram_output.get(), ram_buffer,
+ buffer_length);
+ header_file_count++;
+ }
+ }
+ auto header_len = ram_output->getFilePointer();
+ ram_output->close();
+ ram_dir.deleteFile(idx_name.c_str());
+ ram_dir.close();
+
+ auto* out_dir = DorisFSDirectoryFactory::getDirectory(_fs,
idx_path.c_str());
+
+ auto* out = out_dir->createOutput(idx_name.c_str());
+ if (out == nullptr) {
+ LOG(WARNING) << "Write compound file error: CompoundDirectory
output is nullptr.";
+ _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error");
+ }
+ std::unique_ptr<lucene::store::IndexOutput> output(out);
+ size_t start = output->getFilePointer();
+ output->writeVInt(file_count);
+ // write file entries
+ int64_t data_offset = header_len;
+ uint8_t header_buffer[buffer_length];
+ for (int i = 0; i < sorted_files.size(); ++i) {
+ auto file = sorted_files[i];
+ output->writeString(file.filename); // FileName
+ // DataOffset
+ if (i < header_file_count) {
+ // file data write in header, so we set its offset to -1.
+ output->writeLong(-1);
+ } else {
+ output->writeLong(data_offset);
+ }
+ output->writeLong(file.filesize); // FileLength
+ if (i < header_file_count) {
+ // append data
+ copyFile(file.filename.c_str(), directory.get(),
output.get(), header_buffer,
+ buffer_length);
+ } else {
+ data_offset += file.filesize;
+ }
+ }
+ // write rest files' data
+ uint8_t data_buffer[buffer_length];
+ for (int i = header_file_count; i < sorted_files.size(); ++i) {
+ auto file = sorted_files[i];
+ copyFile(file.filename.c_str(), directory.get(), output.get(),
data_buffer,
+ buffer_length);
+ }
+ out_dir->close();
+ // NOTE: need to decrease ref count, but not to delete here,
+ // because index cache may get the same directory from DIRECTORIES
+ _CLDECDELETE(out_dir)
+ auto compound_file_size = output->getFilePointer() - start;
+ output->close();
+ //LOG(INFO) << (idx_path / idx_name).c_str() << " size:" <<
compound_file_size;
+ total_size += compound_file_size;
+ } catch (CLuceneError& err) {
+ LOG(ERROR) << "CLuceneError occur when close idx file "
+ <<
InvertedIndexDescriptor::get_index_file_path_v1(_index_path_prefix,
+
index_id, index_suffix)
+ << " error msg: " << err.what();
+
+ throw err;
+ }
+ }
+ return total_size;
+}
+
+size_t InvertedIndexFileWriter::write_v2() {
// Create the output stream to write the compound file
int64_t current_offset = headerLength();
- io::Path index_path
{InvertedIndexDescriptor::get_index_path_v2(_index_path_prefix)};
+ io::Path index_path
{InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)};
auto* out_dir = DorisFSDirectoryFactory::getDirectory(_fs,
index_path.parent_path().c_str());
@@ -233,8 +409,7 @@ size_t InvertedIndexFileWriter::write() {
auto* dir = std::get<3>(info);
// Write the actual file data
- DorisCompoundFileWriter::copyFile(file.c_str(), dir,
compound_file_output.get(),
- header_buffer, buffer_length);
+ copyFile(file.c_str(), dir, compound_file_output.get(), header_buffer,
buffer_length);
}
out_dir->close();
@@ -245,177 +420,4 @@ size_t InvertedIndexFileWriter::write() {
compound_file_output->close();
return compound_file_size;
}
-
-DorisCompoundFileWriter::DorisCompoundFileWriter(CL_NS(store)::Directory* dir)
{
- if (dir == nullptr) {
- _CLTHROWA(CL_ERR_NullPointer, "directory cannot be null");
- }
-
- directory = dir;
-}
-
-CL_NS(store)::Directory* DorisCompoundFileWriter::getDirectory() {
- return directory;
-}
-
-void DorisCompoundFileWriter::sort_files(std::vector<FileInfo>& file_infos) {
- auto file_priority = [](const std::string& filename) {
- if (filename.find("segments") != std::string::npos) return 1;
- if (filename.find("fnm") != std::string::npos) return 2;
- if (filename.find("tii") != std::string::npos) return 3;
- return 4; // Other files
- };
-
- std::sort(file_infos.begin(), file_infos.end(), [&](const FileInfo& a,
const FileInfo& b) {
- int32_t priority_a = file_priority(a.filename);
- int32_t priority_b = file_priority(b.filename);
- if (priority_a != priority_b) return priority_a < priority_b;
- return a.filesize < b.filesize;
- });
-}
-
-size_t DorisCompoundFileWriter::writeCompoundFile() {
- // list files in current dir
- std::vector<std::string> files;
- directory->list(&files);
- // remove write.lock file
- auto it = std::find(files.begin(), files.end(),
DorisFSDirectory::WRITE_LOCK_FILE);
- if (it != files.end()) {
- files.erase(it);
- }
-
- std::vector<FileInfo> sorted_files;
- for (auto file : files) {
- FileInfo file_info;
- file_info.filename = file;
- file_info.filesize = directory->fileLength(file.c_str());
- sorted_files.emplace_back(std::move(file_info));
- }
- sort_files(sorted_files);
-
- int32_t file_count = sorted_files.size();
-
- io::Path cfs_path(((DorisFSDirectory*)directory)->getCfsDirName());
- auto idx_path = cfs_path.parent_path();
- std::string idx_name =
- std::string(cfs_path.stem().c_str()) +
DorisFSDirectory::COMPOUND_FILE_EXTENSION;
- // write file entries to ram directory to get header length
- lucene::store::RAMDirectory ram_dir;
- auto* out_idx = ram_dir.createOutput(idx_name.c_str());
- if (out_idx == nullptr) {
- LOG(WARNING) << "Write compound file error: RAMDirectory output is
nullptr.";
- _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error");
- }
-
- std::unique_ptr<lucene::store::IndexOutput> ram_output(out_idx);
- ram_output->writeVInt(file_count);
- // write file entries in ram directory
- // number of files, which data are in header
- int header_file_count = 0;
- int64_t header_file_length = 0;
- const int64_t buffer_length = 16384;
- uint8_t ram_buffer[buffer_length];
- for (auto file : sorted_files) {
- ram_output->writeString(file.filename); // file name
- ram_output->writeLong(0); // data offset
- ram_output->writeLong(file.filesize); // file length
- header_file_length += file.filesize;
- if (header_file_length <= DorisFSDirectory::MAX_HEADER_DATA_SIZE) {
- copyFile(file.filename.c_str(), directory, ram_output.get(),
ram_buffer, buffer_length);
- header_file_count++;
- }
- }
- auto header_len = ram_output->getFilePointer();
- ram_output->close();
- ram_dir.deleteFile(idx_name.c_str());
- ram_dir.close();
-
- auto compound_fs = ((DorisFSDirectory*)directory)->getCompoundFileSystem();
- auto* out_dir = DorisFSDirectoryFactory::getDirectory(compound_fs,
idx_path.c_str());
-
- auto* out = out_dir->createOutput(idx_name.c_str());
- if (out == nullptr) {
- LOG(WARNING) << "Write compound file error: CompoundDirectory output
is nullptr.";
- _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error");
- }
- std::unique_ptr<lucene::store::IndexOutput> output(out);
- size_t start = output->getFilePointer();
- output->writeVInt(file_count);
- // write file entries
- int64_t data_offset = header_len;
- uint8_t header_buffer[buffer_length];
- for (int i = 0; i < sorted_files.size(); ++i) {
- auto file = sorted_files[i];
- output->writeString(file.filename); // FileName
- // DataOffset
- if (i < header_file_count) {
- // file data write in header, so we set its offset to -1.
- output->writeLong(-1);
- } else {
- output->writeLong(data_offset);
- }
- output->writeLong(file.filesize); // FileLength
- if (i < header_file_count) {
- // append data
- copyFile(file.filename.c_str(), directory, output.get(),
header_buffer, buffer_length);
- } else {
- data_offset += file.filesize;
- }
- }
- // write rest files' data
- uint8_t data_buffer[buffer_length];
- for (int i = header_file_count; i < sorted_files.size(); ++i) {
- auto file = sorted_files[i];
- copyFile(file.filename.c_str(), directory, output.get(), data_buffer,
buffer_length);
- }
- out_dir->close();
- // NOTE: need to decrease ref count, but not to delete here,
- // because index cache may get the same directory from DIRECTORIES
- _CLDECDELETE(out_dir)
- auto compound_file_size = output->getFilePointer() - start;
- output->close();
- //LOG(INFO) << (idx_path / idx_name).c_str() << " size:" <<
compound_file_size;
- return compound_file_size;
-}
-
-void DorisCompoundFileWriter::copyFile(const char* fileName,
lucene::store::Directory* dir,
- lucene::store::IndexOutput* output,
uint8_t* buffer,
- int64_t bufferLength) {
- lucene::store::IndexInput* tmp = nullptr;
- CLuceneError err;
- if (!dir->openInput(fileName, tmp, err)) {
- throw err;
- }
-
- std::unique_ptr<lucene::store::IndexInput> input(tmp);
- int64_t start_ptr = output->getFilePointer();
- int64_t length = input->length();
- int64_t remainder = length;
- int64_t chunk = bufferLength;
-
- while (remainder > 0) {
- int64_t len = std::min(std::min(chunk, length), remainder);
- input->readBytes(buffer, len);
- output->writeBytes(buffer, len);
- remainder -= len;
- }
- if (remainder != 0) {
- std::ostringstream errMsg;
- errMsg << "Non-zero remainder length after copying: " << remainder <<
" (id: " << fileName
- << ", length: " << length << ", buffer size: " << chunk << ")";
- err.set(CL_ERR_IO, errMsg.str().c_str());
- throw err;
- }
-
- int64_t end_ptr = output->getFilePointer();
- int64_t diff = end_ptr - start_ptr;
- if (diff != length) {
- std::ostringstream errMsg;
- errMsg << "Difference in the output file offsets " << diff
- << " does not match the original file length " << length;
- err.set(CL_ERR_IO, errMsg.str().c_str());
- throw err;
- }
- input->close();
-}
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
index dc0df8c065f..75e4a7545e3 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
@@ -36,27 +36,10 @@ class DorisFSDirectory;
using InvertedIndexDirectoryMap =
std::map<std::pair<int64_t, std::string>,
std::unique_ptr<lucene::store::Directory>>;
-class DorisCompoundFileWriter : LUCENE_BASE {
+class FileInfo {
public:
- DorisCompoundFileWriter() = default;
- DorisCompoundFileWriter(CL_NS(store)::Directory* dir);
- ~DorisCompoundFileWriter() override = default;
- /** Returns the directory of the compound file. */
- CL_NS(store)::Directory* getDirectory();
- virtual size_t writeCompoundFile();
- static void copyFile(const char* fileName, lucene::store::Directory* dir,
- lucene::store::IndexOutput* output, uint8_t* buffer,
int64_t bufferLength);
-
-private:
- class FileInfo {
- public:
- std::string filename;
- int32_t filesize;
- };
-
- void sort_files(std::vector<FileInfo>& file_infos);
-
- CL_NS(store)::Directory* directory = nullptr;
+ std::string filename;
+ int32_t filesize;
};
class InvertedIndexFileWriter {
@@ -74,12 +57,15 @@ public:
Status delete_index(const TabletIndex* index_meta);
Status initialize(InvertedIndexDirectoryMap& indices_dirs);
~InvertedIndexFileWriter() = default;
- size_t write();
+ size_t write_v2();
+ size_t write_v1();
Status close();
size_t headerLength();
- std::string get_index_file_path() const;
size_t get_index_file_size() const { return _file_size; }
const io::FileSystemSPtr& get_fs() const { return _fs; }
+ void sort_files(std::vector<FileInfo>& file_infos);
+ void copyFile(const char* fileName, lucene::store::Directory* dir,
+ lucene::store::IndexOutput* output, uint8_t* buffer, int64_t
bufferLength);
private:
InvertedIndexDirectoryMap _indices_dirs;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
index 1c6e75aa97e..499a3e41d9a 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
@@ -82,7 +82,6 @@
namespace doris::segment_v2 {
const char* const DorisFSDirectory::WRITE_LOCK_FILE = "write.lock";
-const char* const DorisFSDirectory::COMPOUND_FILE_EXTENSION = ".idx";
class DorisFSDirectory::FSIndexOutput : public
lucene::store::BufferedIndexOutput {
private:
@@ -341,24 +340,10 @@ DorisFSDirectory::DorisFSDirectory() {
this->lockFactory = nullptr;
}
-void DorisFSDirectory::init(const io::FileSystemSPtr& _fs, const char* _path,
- bool use_compound_file_writer,
lucene::store::LockFactory* lock_factory,
- const io::FileSystemSPtr& cfs, const char*
cfs_path) {
- fs = _fs;
- directory = _path;
- useCompoundFileWriter = use_compound_file_writer;
-
- if (cfs == nullptr) {
- compound_fs = fs;
- } else {
- compound_fs = cfs;
- }
- if (cfs_path != nullptr) {
- cfs_directory = cfs_path;
- } else {
- cfs_directory = _path;
- }
-
+void DorisFSDirectory::init(const io::FileSystemSPtr& fs, const char* path,
+ lucene::store::LockFactory* lock_factory) {
+ _fs = fs;
+ directory = path;
if (lock_factory == nullptr) {
lock_factory = _CLNEW lucene::store::NoLockFactory();
}
@@ -366,11 +351,11 @@ void DorisFSDirectory::init(const io::FileSystemSPtr&
_fs, const char* _path,
lucene::store::Directory::setLockFactory(lock_factory);
// It's fail checking directory existence in S3.
- if (fs->type() == io::FileSystemType::S3) {
+ if (_fs->type() == io::FileSystemType::S3) {
return;
}
bool exists = false;
- LOG_AND_THROW_IF_ERROR(fs->exists(directory, &exists),
+ LOG_AND_THROW_IF_ERROR(_fs->exists(directory, &exists),
"Doris compound directory init IO error");
if (!exists) {
auto e = "Doris compound directory init error: " + directory + " is
not a directory";
@@ -401,7 +386,7 @@ bool DorisFSDirectory::list(std::vector<std::string>*
names) const {
priv_getFN(fl, "");
std::vector<io::FileInfo> files;
bool exists;
- LOG_AND_THROW_IF_ERROR(fs->list(fl, true, &files, &exists), "List file IO
error");
+ LOG_AND_THROW_IF_ERROR(_fs->list(fl, true, &files, &exists), "List file IO
error");
for (auto& file : files) {
names->push_back(file.file_name);
}
@@ -413,14 +398,10 @@ bool DorisFSDirectory::fileExists(const char* name) const
{
char fl[CL_MAX_DIR];
priv_getFN(fl, name);
bool exists = false;
- LOG_AND_THROW_IF_ERROR(fs->exists(fl, &exists), "File exists IO error");
+ LOG_AND_THROW_IF_ERROR(_fs->exists(fl, &exists), "File exists IO error");
return exists;
}
-const char* DorisFSDirectory::getCfsDirName() const {
- return cfs_directory.c_str();
-}
-
const std::string& DorisFSDirectory::getDirName() const {
return directory;
}
@@ -443,7 +424,7 @@ void DorisFSDirectory::touchFile(const char* name) {
snprintf(buffer, CL_MAX_DIR, "%s%s%s", directory.c_str(), PATH_DELIMITERA,
name);
io::FileWriterPtr tmp_writer;
- LOG_AND_THROW_IF_ERROR(fs->create_file(buffer, &tmp_writer), "Touch file
IO error");
+ LOG_AND_THROW_IF_ERROR(_fs->create_file(buffer, &tmp_writer), "Touch file
IO error");
}
int64_t DorisFSDirectory::fileLength(const char* name) const {
@@ -451,7 +432,7 @@ int64_t DorisFSDirectory::fileLength(const char* name)
const {
char buffer[CL_MAX_DIR];
priv_getFN(buffer, name);
int64_t size = -1;
- LOG_AND_THROW_IF_ERROR(fs->file_size(buffer, &size), "Get file size IO
error");
+ LOG_AND_THROW_IF_ERROR(_fs->file_size(buffer, &size), "Get file size IO
error");
return size;
}
@@ -460,7 +441,7 @@ bool DorisFSDirectory::openInput(const char* name,
lucene::store::IndexInput*& r
CND_PRECONDITION(directory[0] != 0, "directory is not open");
char fl[CL_MAX_DIR];
priv_getFN(fl, name);
- return FSIndexInput::open(fs, fl, ret, error, bufferSize);
+ return FSIndexInput::open(_fs, fl, ret, error, bufferSize);
}
void DorisFSDirectory::close() {}
@@ -469,7 +450,7 @@ bool DorisFSDirectory::doDeleteFile(const char* name) {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
char fl[CL_MAX_DIR];
priv_getFN(fl, name);
- LOG_AND_THROW_IF_ERROR(fs->delete_file(fl), "Delete file IO error");
+ LOG_AND_THROW_IF_ERROR(_fs->delete_file(fl), "Delete file IO error");
return true;
}
@@ -477,7 +458,7 @@ bool DorisFSDirectory::deleteDirectory() {
CND_PRECONDITION(directory[0] != 0, "directory is not open");
char fl[CL_MAX_DIR];
priv_getFN(fl, "");
- LOG_AND_THROW_IF_ERROR(fs->delete_directory(fl),
+ LOG_AND_THROW_IF_ERROR(_fs->delete_directory(fl),
fmt::format("Delete directory {} IO error", fl));
return true;
}
@@ -492,11 +473,11 @@ void DorisFSDirectory::renameFile(const char* from, const
char* to) {
priv_getFN(nu, to);
bool exists = false;
- LOG_AND_THROW_IF_ERROR(fs->exists(nu, &exists), "File exists IO error");
+ LOG_AND_THROW_IF_ERROR(_fs->exists(nu, &exists), "File exists IO error");
if (exists) {
- LOG_AND_THROW_IF_ERROR(fs->delete_directory(nu), fmt::format("Delete
{} IO error", nu));
+ LOG_AND_THROW_IF_ERROR(_fs->delete_directory(nu), fmt::format("Delete
{} IO error", nu));
}
- LOG_AND_THROW_IF_ERROR(fs->rename(old, nu), fmt::format("Rename {} to {}
IO error", old, nu));
+ LOG_AND_THROW_IF_ERROR(_fs->rename(old, nu), fmt::format("Rename {} to {}
IO error", old, nu));
}
lucene::store::IndexOutput* DorisFSDirectory::createOutput(const char* name) {
@@ -504,16 +485,16 @@ lucene::store::IndexOutput*
DorisFSDirectory::createOutput(const char* name) {
char fl[CL_MAX_DIR];
priv_getFN(fl, name);
bool exists = false;
- LOG_AND_THROW_IF_ERROR(fs->exists(fl, &exists), "Create output file exists
IO error");
+ LOG_AND_THROW_IF_ERROR(_fs->exists(fl, &exists), "Create output file
exists IO error");
if (exists) {
- LOG_AND_THROW_IF_ERROR(fs->delete_file(fl),
+ LOG_AND_THROW_IF_ERROR(_fs->delete_file(fl),
fmt::format("Create output delete file {} IO
error", fl));
- LOG_AND_THROW_IF_ERROR(fs->exists(fl, &exists), "Create output file
exists IO error");
+ LOG_AND_THROW_IF_ERROR(_fs->exists(fl, &exists), "Create output file
exists IO error");
assert(!exists);
}
auto* ret = _CLNEW FSIndexOutput();
try {
- ret->init(fs, fl);
+ ret->init(_fs, fl);
} catch (CLuceneError& err) {
ret->close();
_CLDELETE(ret)
@@ -539,24 +520,10 @@ DorisRAMFSDirectory::~DorisRAMFSDirectory() {
_CLDELETE(filesMap);
}
-void DorisRAMFSDirectory::init(const io::FileSystemSPtr& _fs, const char*
_path,
- bool use_compound_file_writer,
- lucene::store::LockFactory* lock_factory,
- const io::FileSystemSPtr& cfs, const char*
cfs_path) {
- fs = _fs;
- directory = _path;
- useCompoundFileWriter = use_compound_file_writer;
-
- if (cfs == nullptr) {
- compound_fs = fs;
- } else {
- compound_fs = cfs;
- }
- if (cfs_path != nullptr) {
- cfs_directory = cfs_path;
- } else {
- cfs_directory = _path;
- }
+void DorisRAMFSDirectory::init(const io::FileSystemSPtr& fs, const char* path,
+ lucene::store::LockFactory* lock_factory) {
+ _fs = fs;
+ directory = path;
lucene::store::Directory::setLockFactory(_CLNEW
lucene::store::SingleInstanceLockFactory());
}
@@ -706,14 +673,9 @@ const char* DorisRAMFSDirectory::getObjectName() const {
return getClassName();
}
-DorisFSDirectory* DorisFSDirectoryFactory::getDirectory(
- const io::FileSystemSPtr& _fs, const char* _file, bool
use_compound_file_writer,
- bool can_use_ram_dir, lucene::store::LockFactory* lock_factory,
- const io::FileSystemSPtr& _cfs, const char* _cfs_file) {
- const char* cfs_file = _cfs_file;
- if (cfs_file == nullptr) {
- cfs_file = _file;
- }
+DorisFSDirectory* DorisFSDirectoryFactory::getDirectory(const
io::FileSystemSPtr& _fs,
+ const char* _file,
bool can_use_ram_dir,
+
lucene::store::LockFactory* lock_factory) {
DorisFSDirectory* dir = nullptr;
if (!_file || !*_file) {
_CLTHROWA(CL_ERR_IO, "Invalid directory");
@@ -735,7 +697,7 @@ DorisFSDirectory* DorisFSDirectoryFactory::getDirectory(
}
dir = _CLNEW DorisFSDirectory();
}
- dir->init(_fs, file, use_compound_file_writer, lock_factory, _cfs,
cfs_file);
+ dir->init(_fs, file, lock_factory);
return dir;
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
index 0478070b4f5..d9069d66ef2 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
@@ -22,7 +22,6 @@
#include <CLucene/store/IndexInput.h>
#include <CLucene/store/IndexOutput.h>
#include <CLucene/store/_RAMDirectory.h>
-#include <stdint.h>
#include <string>
#include <vector>
@@ -46,19 +45,14 @@ namespace segment_v2 {
class CLUCENE_EXPORT DorisFSDirectory : public lucene::store::Directory {
public:
static const char* const WRITE_LOCK_FILE;
- static const char* const COMPOUND_FILE_EXTENSION;
static const int64_t MAX_HEADER_DATA_SIZE = 1024 * 128; // 128k
private:
int filemode;
protected:
mutable std::mutex _this_lock;
- io::FileSystemSPtr fs;
- io::FileSystemSPtr compound_fs;
+ io::FileSystemSPtr _fs;
std::string directory;
- std::string cfs_directory;
- bool useCompoundFileWriter {false};
- size_t compound_file_size = 0;
void priv_getFN(char* buffer, const char* name) const;
/// Removes an existing file in the directory.
@@ -71,14 +65,11 @@ public:
friend class DorisFSDirectory::FSIndexOutput;
friend class DorisFSDirectory::FSIndexInput;
- const io::FileSystemSPtr& getFileSystem() { return fs; }
- const io::FileSystemSPtr& getCompoundFileSystem() { return compound_fs; }
- size_t getCompoundFileSize() const { return compound_file_size; }
+ const io::FileSystemSPtr& getFileSystem() { return _fs; }
~DorisFSDirectory() override;
bool list(std::vector<std::string>* names) const override;
bool fileExists(const char* name) const override;
- const char* getCfsDirName() const;
const std::string& getDirName() const;
int64_t fileModified(const char* name) const override;
int64_t fileLength(const char* name) const override;
@@ -95,10 +86,8 @@ public:
DorisFSDirectory();
- virtual void init(const io::FileSystemSPtr& fs, const char* path, bool
use_compound_file_writer,
- lucene::store::LockFactory* lock_factory = nullptr,
- const io::FileSystemSPtr& compound_fs = nullptr,
- const char* cfs_path = nullptr);
+ virtual void init(const io::FileSystemSPtr& fs, const char* path,
+ lucene::store::LockFactory* lock_factory = nullptr);
};
class CLUCENE_EXPORT DorisRAMFSDirectory : public DorisFSDirectory {
@@ -110,10 +99,8 @@ protected:
// unlike the java Hashtable, FileMap is not synchronized, and all access
must be protected by a lock
FileMap* filesMap;
- void init(const io::FileSystemSPtr& fs, const char* path, bool
use_compound_file_writer,
- lucene::store::LockFactory* lock_factory = nullptr,
- const io::FileSystemSPtr& compound_fs = nullptr,
- const char* cfs_path = nullptr) override;
+ void init(const io::FileSystemSPtr& fs, const char* path,
+ lucene::store::LockFactory* lock_factory = nullptr) override;
public:
int64_t sizeInBytes;
@@ -222,11 +209,8 @@ protected:
class DorisFSDirectoryFactory {
public:
static DorisFSDirectory* getDirectory(const io::FileSystemSPtr& fs, const
char* file,
- bool use_compound_file_writer =
false,
bool can_use_ram_dir = false,
- lucene::store::LockFactory*
lock_factory = nullptr,
- const io::FileSystemSPtr& cfs_fs =
nullptr,
- const char* cfs_file = nullptr);
+ lucene::store::LockFactory*
lock_factory = nullptr);
};
} // namespace segment_v2
} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 3639bff05c4..757b8539330 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -193,7 +193,7 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
bool owned_dir = false;
try {
// try to get query bitmap result from cache and return immediately on
cache hit
- auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {
index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY,
"null_bitmap"};
auto* cache = InvertedIndexQueryCache::instance();
@@ -242,7 +242,7 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
Status InvertedIndexReader::handle_searcher_cache(
InvertedIndexCacheHandle* inverted_index_cache_handle,
OlapReaderStatistics* stats) {
- auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
inverted_index_cache_handle)) {
@@ -316,7 +316,7 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
try {
InvertedIndexQueryInfo query_info;
InvertedIndexQueryCache::CacheKey cache_key;
- auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
cache_key = {index_file_key, column_name, query_type, search_str};
@@ -472,7 +472,7 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
[](lucene::index::Term* term) { _CLDECDELETE(term); }};
std::unique_ptr<lucene::search::Query> query;
- auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
// try to get query bitmap result from cache and return immediately on
cache hit
InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name,
query_type,
@@ -722,7 +722,7 @@ Status BkdIndexReader::try_query(OlapReaderStatistics*
stats, const std::string&
std::string query_str;
_value_key_coder->full_encode_ascending(query_value, &query_str);
- auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {index_file_key,
column_name, query_type,
query_str};
auto* cache = InvertedIndexQueryCache::instance();
@@ -762,7 +762,7 @@ Status BkdIndexReader::query(OlapReaderStatistics* stats,
RuntimeState* runtime_
std::string query_str;
_value_key_coder->full_encode_ascending(query_value, &query_str);
- auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {index_file_key,
column_name, query_type,
query_str};
auto* cache = InvertedIndexQueryCache::instance();
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 9f51098a052..50d376d0aed 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -538,8 +538,6 @@ public:
return 0;
}
- int64_t file_size() const override { return _dir->getCompoundFileSize(); }
-
void write_null_bitmap(lucene::store::IndexOutput* null_bitmap_out) {
// write null_bitmap file
_null_bitmap.runOptimize();
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.h
b/be/src/olap/rowset/segment_v2/inverted_index_writer.h
index 06bc960bc33..33fd9d4689f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.h
@@ -69,8 +69,6 @@ public:
virtual int64_t size() const = 0;
- virtual int64_t file_size() const = 0;
-
virtual void close_on_error() = 0;
// check if the column is valid for inverted index, some columns
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index 69118eb2782..8d862528de6 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -137,8 +137,8 @@ Status Segment::_open() {
Status Segment::_open_inverted_index() {
_inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
_fs,
- std::string {
-
InvertedIndexDescriptor::get_index_path_prefix(_file_reader->path().native())},
+ std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
+ _file_reader->path().native())},
_tablet_schema->get_inverted_index_storage_format());
bool open_idx_file_cache = true;
auto st =
_inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 820afaa2103..de4afb39235 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -137,7 +137,7 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer,
uint32_t segment_id,
if (_tablet_schema->has_inverted_index()) {
_inverted_index_file_writer =
std::make_unique<InvertedIndexFileWriter>(
_opts.rowset_ctx->fs(),
- std::string {InvertedIndexDescriptor::get_index_path_prefix(
+ std::string
{InvertedIndexDescriptor::get_index_file_path_prefix(
_opts.rowset_ctx->segment_path(segment_id))},
_opts.rowset_ctx->rowset_id.to_string(), segment_id,
_tablet_schema->get_inverted_index_storage_format());
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index d763543e7d1..f24ca36944a 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -111,7 +111,7 @@
VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32
if (_tablet_schema->has_inverted_index()) {
_inverted_index_file_writer =
std::make_unique<InvertedIndexFileWriter>(
_opts.rowset_ctx->fs(),
- std::string {InvertedIndexDescriptor::get_index_path_prefix(
+ std::string
{InvertedIndexDescriptor::get_index_file_path_prefix(
_opts.rowset_ctx->segment_path(segment_id))},
_opts.rowset_ctx->rowset_id.to_string(), segment_id,
_tablet_schema->get_inverted_index_storage_format());
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index dc597ae470d..eab49997599 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -688,8 +688,9 @@ Status SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_tablet
}
} else {
if (tablet_schema.has_inverted_index()) {
- auto index_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(segment_file_path));
+ auto index_file =
InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ segment_file_path));
auto snapshot_segment_index_file_path =
fmt::format("{}/{}_{}.binlog-index",
schema_full_path, rowset_id,
segment_index);
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index 28ab654b413..4ed8f1e31af 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -113,8 +113,8 @@ Status IndexBuilder::update_inverted_index_info() {
auto seg_path =
local_segment_path(_tablet->tablet_path(),
input_rowset->rowset_id().to_string(), seg_id);
- auto index_path =
InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(seg_path),
+ auto index_path =
InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path),
index_meta->index_id(),
index_meta->get_index_suffix());
int64_t index_size = 0;
RETURN_IF_ERROR(fs->file_size(index_path,
&index_size));
@@ -204,7 +204,7 @@ Status IndexBuilder::update_inverted_index_info() {
auto seg_path = DORIS_TRY(input_rowset->segment_path(seg_id));
auto idx_file_reader =
std::make_unique<InvertedIndexFileReader>(
context.fs(),
- std::string
{InvertedIndexDescriptor::get_index_path_prefix(seg_path)},
+ std::string
{InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)},
output_rs_tablet_schema->get_inverted_index_storage_format());
auto st = idx_file_reader->init();
if (!st.ok() &&
!st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>()) {
@@ -251,7 +251,7 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
const auto& output_rs_tablet_schema =
output_rowset_meta->tablet_schema();
if (output_rs_tablet_schema->get_inverted_index_storage_format() !=
InvertedIndexStorageFormatPB::V1) {
- const auto& fs = io::global_local_filesystem();
+ const auto& fs = output_rowset_meta->fs();
const auto& output_rowset_schema =
output_rowset_meta->tablet_schema();
size_t inverted_index_size = 0;
@@ -266,7 +266,7 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
auto dirs =
DORIS_TRY(idx_file_reader_iter->second->get_all_directories());
std::string index_path_prefix {
-
InvertedIndexDescriptor::get_index_path_prefix(local_segment_path(
+
InvertedIndexDescriptor::get_index_file_path_prefix(local_segment_path(
_tablet->tablet_path(),
output_rowset_meta->rowset_id().to_string(),
seg_ptr->id()))};
@@ -283,12 +283,10 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
std::move(inverted_index_file_writer));
}
for (auto&& [seg_id, inverted_index_writer] :
_inverted_index_file_writers) {
- LOG(INFO) << "close inverted index file "
- << inverted_index_writer->get_index_file_path();
auto st = inverted_index_writer->close();
if (!st.ok()) {
- LOG(ERROR) << "close inverted index file "
- << inverted_index_writer->get_index_file_path()
<< " error:" << st;
+ LOG(ERROR) << "close inverted_index_writer error:" << st;
+ return st;
}
inverted_index_size +=
inverted_index_writer->get_index_file_size();
}
@@ -309,7 +307,7 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
size_t inverted_index_size = 0;
for (auto& seg_ptr : segments) {
std::string index_path_prefix {
-
InvertedIndexDescriptor::get_index_path_prefix(local_segment_path(
+
InvertedIndexDescriptor::get_index_file_path_prefix(local_segment_path(
_tablet->tablet_path(),
output_rowset_meta->rowset_id().to_string(),
seg_ptr->id()))};
std::vector<ColumnId> return_columns;
@@ -317,8 +315,8 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
_olap_data_convertor->reserve(_alter_inverted_indexes.size());
std::unique_ptr<InvertedIndexFileWriter>
inverted_index_file_writer = nullptr;
- if (output_rowset_schema->get_inverted_index_storage_format() !=
- InvertedIndexStorageFormatPB::V1) {
+ if (output_rowset_schema->get_inverted_index_storage_format() >=
+ InvertedIndexStorageFormatPB::V2) {
auto idx_file_reader_iter = _inverted_index_file_readers.find(
std::make_pair(output_rowset_meta->rowset_id().to_string(), seg_ptr->id()));
if (idx_file_reader_iter ==
_inverted_index_file_readers.end()) {
@@ -440,12 +438,10 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
_olap_data_convertor->reset();
}
for (auto&& [seg_id, inverted_index_file_writer] :
_inverted_index_file_writers) {
- LOG(INFO) << "close inverted index file "
- << inverted_index_file_writer->get_index_file_path();
auto st = inverted_index_file_writer->close();
if (!st.ok()) {
- LOG(ERROR) << "close inverted index file "
- <<
inverted_index_file_writer->get_index_file_path() << " error:" << st;
+ LOG(ERROR) << "close inverted_index_writer error:" << st;
+ return st;
}
inverted_index_size +=
inverted_index_file_writer->get_index_file_size();
}
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index 1b2034a0e21..0ff9608ae4a 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -351,8 +351,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
auto segment_path =
local_segment_path(local_tablet->tablet_path(),
rowset_meta->rowset_id().to_string(), segment_index);
-
segment_index_file_names.push_back(InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(segment_path), index_id,
+
segment_index_file_names.push_back(InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(segment_path), index_id,
index.get_index_suffix()));
status = HttpClient::execute_with_retry(max_retry, 1,
@@ -389,8 +389,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
auto segment_path =
local_segment_path(local_tablet->tablet_path(),
rowset_meta->rowset_id().to_string(), segment_index);
-
segment_index_file_names.push_back(InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(segment_path)));
+
segment_index_file_names.push_back(InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)));
status = HttpClient::execute_with_retry(max_retry, 1,
get_segment_index_file_size_cb);
diff --git a/be/src/service/internal_service.cpp
b/be/src/service/internal_service.cpp
index 18113bd7c8c..c4ddf7ef679 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -1784,25 +1784,31 @@ void
PInternalServiceImpl::request_slave_tablet_pull_rowset(
std::string remote_inverted_index_file;
std::string local_inverted_index_file;
std::string remote_inverted_index_file_url;
- if (tablet_scheme->get_inverted_index_storage_format() !=
+ if (tablet_scheme->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
- remote_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(remote_file_path));
+ remote_inverted_index_file =
+
InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ remote_file_path),
+ index_id, suffix_path);
remote_inverted_index_file_url = construct_url(
get_host_port(host, http_port), token,
remote_inverted_index_file);
- local_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v2(
-
InvertedIndexDescriptor::get_index_path_prefix(local_file_path));
- } else {
- remote_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(remote_file_path),
+ local_inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ local_file_path),
index_id, suffix_path);
+ } else {
+ remote_inverted_index_file =
+
InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ remote_file_path));
remote_inverted_index_file_url = construct_url(
get_host_port(host, http_port), token,
remote_inverted_index_file);
- local_inverted_index_file =
InvertedIndexDescriptor::get_index_path_v1(
-
InvertedIndexDescriptor::get_index_path_prefix(local_file_path),
- index_id, suffix_path);
+ local_inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v2(
+
InvertedIndexDescriptor::get_index_file_path_prefix(
+ local_file_path));
}
st = download_file_action(remote_inverted_index_file_url,
local_inverted_index_file,
estimate_timeout, size);
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
index 2c7a256f654..7ebf89300da 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
@@ -112,11 +112,11 @@ public:
void test_string(std::string_view rowset_id, int seg_id, Field* field) {
EXPECT_TRUE(field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY);
- std::string index_path_prefix
{InvertedIndexDescriptor::get_index_path_prefix(
+ std::string index_path_prefix
{InvertedIndexDescriptor::get_index_file_path_prefix(
local_segment_path(kTestDir, rowset_id, seg_id))};
int index_id = 26033;
std::string index_path =
- InvertedIndexDescriptor::get_index_path_v1(index_path_prefix,
index_id, "");
+
InvertedIndexDescriptor::get_index_file_path_v1(index_path_prefix, index_id,
"");
auto fs = io::global_local_filesystem();
io::FileWriterPtr file_writer;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
