This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 89215306d32 [improve](load) add switch for vertical segment writer
(#26996)
89215306d32 is described below
commit 89215306d325f386e1ba3cfc91e59c0ec79a342f
Author: Kaijie Chen <c...@apache.org>
AuthorDate: Wed Nov 15 08:19:12 2023 +0800
[improve](load) add switch for vertical segment writer (#26996)
---
be/src/common/config.cpp | 3 +++
be/src/common/config.h | 3 +++
be/src/olap/rowset/segment_creator.cpp | 15 +++++++++++----
3 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index a9b0848984b..4d58b1fc941 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -351,6 +351,9 @@ DEFINE_Int32(vertical_compaction_max_row_source_memory_mb,
"200");
// In vertical compaction, max dest segment file size
DEFINE_mInt64(vertical_compaction_max_segment_size, "268435456");
+// If enabled, segments will be flushed column by column
+DEFINE_mBool(enable_vertical_segment_writer, "true");
+
// In ordered data compaction, min segment size for input rowset
DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index b6e911438ec..6317fc76366 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -404,6 +404,9 @@ DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
// In vertical compaction, max dest segment file size
DECLARE_mInt64(vertical_compaction_max_segment_size);
+// If enabled, segments will be flushed column by column
+DECLARE_mBool(enable_vertical_segment_writer);
+
// In ordered data compaction, min segment size for input rowset
DECLARE_mInt32(ordered_data_compaction_min_segment_size);
diff --git a/be/src/olap/rowset/segment_creator.cpp
b/be/src/olap/rowset/segment_creator.cpp
index f40d899bdce..5eabc2cc237 100644
--- a/be/src/olap/rowset/segment_creator.cpp
+++ b/be/src/olap/rowset/segment_creator.cpp
@@ -50,11 +50,18 @@ Status SegmentFlusher::flush_single_block(const
vectorized::Block* block, int32_
if (block->rows() == 0) {
return Status::OK();
}
- std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
bool no_compression = block->bytes() <=
config::segment_compression_threshold_kb * 1024;
- RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression,
flush_schema));
- RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
- RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
+ if (config::enable_vertical_segment_writer) {
+ std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
+ RETURN_IF_ERROR(_create_segment_writer(writer, segment_id,
no_compression, flush_schema));
+ RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
+ RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
+ } else {
+ std::unique_ptr<segment_v2::SegmentWriter> writer;
+ RETURN_IF_ERROR(_create_segment_writer(writer, segment_id,
no_compression, flush_schema));
+ RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
+ RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
+ }
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
