airborne12 commented on code in PR #41625:
URL: https://github.com/apache/doris/pull/41625#discussion_r1820233998
##########
be/src/olap/rowset/beta_rowset_writer.cpp:
##########
@@ -891,13 +947,36 @@ Status BaseBetaRowsetWriter::create_file_writer(uint32_t
segment_id, io::FileWri
fmt::format("failed to create file = {}, file type = {}",
segment_path, file_type));
}
+Status BaseBetaRowsetWriter::create_inverted_index_file_writer(
+ uint32_t segment_id, InvertedIndexFileWriterPtr* index_file_writer) {
+
RETURN_IF_ERROR(RowsetWriter::create_inverted_index_file_writer(segment_id,
index_file_writer));
+ // used for inverted index format v1
+
(*index_file_writer)->set_file_writer_opts(_context.get_file_writer_options());
+ return Status::OK();
+}
+
Status BetaRowsetWriter::_create_segment_writer_for_segcompaction(
std::unique_ptr<segment_v2::SegmentWriter>* writer, int64_t begin,
int64_t end) {
DCHECK(begin >= 0 && end >= 0);
std::string path =
BetaRowset::local_segment_path_segcompacted(_context.tablet_path,
_context.rowset_id, begin, end);
io::FileWriterPtr file_writer;
RETURN_IF_ERROR(_create_file_writer(path, file_writer));
+ std::string prefix = std::string
{InvertedIndexDescriptor::get_index_file_path_prefix(path)};
Review Comment:
move line 965 and 966 down to 973
##########
be/src/olap/compaction.cpp:
##########
@@ -737,40 +666,12 @@ Status Compaction::do_inverted_index_compaction() {
}
}
- std::vector<InvertedIndexFileInfo>
all_inverted_index_file_info(dest_segment_num);
- uint64_t inverted_index_file_size = 0;
- for (int seg_id = 0; seg_id < dest_segment_num; ++seg_id) {
- auto inverted_index_file_writer =
inverted_index_file_writers[seg_id].get();
- if (Status st = inverted_index_file_writer->close(); !st.ok()) {
- status = Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(st.msg());
- } else {
- inverted_index_file_size +=
inverted_index_file_writer->get_index_file_total_size();
- inverted_index_file_size -= compacted_idx_file_size[seg_id];
- }
- all_inverted_index_file_info[seg_id] =
inverted_index_file_writer->get_index_file_info();
- }
// check index compaction status. If status is not ok, we should return
error and end this compaction round.
if (!status.ok()) {
return status;
}
-
- // index compaction should update total disk size and index disk size
-
_output_rowset->rowset_meta()->set_data_disk_size(_output_rowset->data_disk_size()
+
-
inverted_index_file_size);
-
_output_rowset->rowset_meta()->set_total_disk_size(_output_rowset->data_disk_size()
+
-
inverted_index_file_size);
-
_output_rowset->rowset_meta()->set_index_disk_size(_output_rowset->index_disk_size()
+
-
inverted_index_file_size);
-
-
_output_rowset->rowset_meta()->update_inverted_index_files_info(all_inverted_index_file_info);
- COUNTER_UPDATE(_output_rowset_data_size_counter,
_output_rowset->data_disk_size());
-
LOG(INFO) << "succeed to do index compaction"
- << ". tablet=" << _tablet->tablet_id() << ", input row number="
<< _input_row_num
- << ", output row number=" << _output_rowset->num_rows()
- << ", input_rowset_size=" << _input_rowsets_size
- << ", output_rowset_size=" << _output_rowset->data_disk_size()
- << ", inverted index file size=" << inverted_index_file_size
+ << ". tablet=" << _tablet->tablet_id()
Review Comment:
Why all other info erased?
##########
be/src/olap/rowset/rowset_writer.h:
##########
@@ -95,6 +96,24 @@ class RowsetWriter {
return Status::NotSupported("RowsetWriter does not support
create_file_writer");
}
+ virtual Status create_inverted_index_file_writer(
+ uint32_t segment_id, InvertedIndexFileWriterPtr*
index_file_writer) {
+ std::string segment_prefix
{InvertedIndexDescriptor::get_index_file_path_prefix(
Review Comment:
also as above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
