[doris] 01/08: [fix](segcompaction) fix segcompaction failed for newly created segment (#15022) (#15023)

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 1be148080dd4f4b62578cbb01886c1e351633721
Author: zhengyu <freeman.zhang1...@gmail.com>
AuthorDate: Mon Dec 19 14:17:58 2022 +0800

    [fix](segcompaction) fix segcompaction failed for newly created segment 
(#15022) (#15023)
    
    Currently, newly created segment could be chosen to be compaction
    candidate, which is prone to bugs and segment file open failures. We
    should skip last (maybe active) segment while doing segcompaction.
---
 be/src/olap/rowset/beta_rowset_writer.cpp | 9 +++++----
 be/src/olap/rowset/beta_rowset_writer.h   | 3 ++-
 be/test/olap/segcompaction_test.cpp       | 1 +
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp 
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 512e4d85b1..687e8bac35 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -382,12 +382,12 @@ void 
BetaRowsetWriter::compact_segments(SegCompactionCandidatesSharedPtr segment
 }
 
 Status BetaRowsetWriter::_load_noncompacted_segments(
-        std::vector<segment_v2::SegmentSharedPtr>* segments) {
+        std::vector<segment_v2::SegmentSharedPtr>* segments, size_t num) {
     auto fs = _rowset_meta->fs();
     if (!fs) {
         return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED);
     }
-    for (int seg_id = _segcompacted_point; seg_id < _num_segment; ++seg_id) {
+    for (int seg_id = _segcompacted_point; seg_id < num; ++seg_id) {
         auto seg_path =
                 BetaRowset::segment_file_path(_context.rowset_dir, 
_context.rowset_id, seg_id);
         auto cache_path =
@@ -414,7 +414,8 @@ Status BetaRowsetWriter::_load_noncompacted_segments(
 Status BetaRowsetWriter::_find_longest_consecutive_small_segment(
         SegCompactionCandidatesSharedPtr segments) {
     std::vector<segment_v2::SegmentSharedPtr> all_segments;
-    RETURN_NOT_OK(_load_noncompacted_segments(&all_segments));
+    // subtract one to skip last (maybe active) segment
+    RETURN_NOT_OK(_load_noncompacted_segments(&all_segments, _num_segment - 
1));
 
     if (VLOG_DEBUG_IS_ON) {
         vlog_buffer.clear();
@@ -472,7 +473,7 @@ Status 
BetaRowsetWriter::_get_segcompaction_candidates(SegCompactionCandidatesSh
         VLOG_DEBUG << "segcompaction last few segments";
         // currently we only rename remaining segments to reduce wait time
         // so that transaction can be committed ASAP
-        RETURN_NOT_OK(_load_noncompacted_segments(segments.get()));
+        RETURN_NOT_OK(_load_noncompacted_segments(segments.get(), 
_num_segment));
         for (int i = 0; i < segments->size(); ++i) {
             
RETURN_NOT_OK(_rename_compacted_segment_plain(_segcompacted_point++));
         }
diff --git a/be/src/olap/rowset/beta_rowset_writer.h 
b/be/src/olap/rowset/beta_rowset_writer.h
index 5f4077d4b5..04e13810ae 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -110,7 +110,8 @@ private:
     Status _delete_original_segments(uint32_t begin, uint32_t end);
     Status _rename_compacted_segments(int64_t begin, int64_t end);
     Status _rename_compacted_segment_plain(uint64_t seg_id);
-    Status 
_load_noncompacted_segments(std::vector<segment_v2::SegmentSharedPtr>* 
segments);
+    Status 
_load_noncompacted_segments(std::vector<segment_v2::SegmentSharedPtr>* segments,
+                                       size_t num);
     Status 
_find_longest_consecutive_small_segment(SegCompactionCandidatesSharedPtr 
segments);
     Status _get_segcompaction_candidates(SegCompactionCandidatesSharedPtr& 
segments, bool is_last);
     Status _wait_flying_segcompaction();
diff --git a/be/test/olap/segcompaction_test.cpp 
b/be/test/olap/segcompaction_test.cpp
index 205fa0cbbd..cb9fb010c0 100644
--- a/be/test/olap/segcompaction_test.cpp
+++ b/be/test/olap/segcompaction_test.cpp
@@ -252,6 +252,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
         ls.push_back("10047_3.dat");
         ls.push_back("10047_4.dat");
         ls.push_back("10047_5.dat");
+        ls.push_back("10047_6.dat");
         EXPECT_TRUE(check_dir(ls));
     }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org