yiguolei commented on code in PR #15624:
URL: https://github.com/apache/doris/pull/15624#discussion_r1062143853
##########
be/src/vec/common/sort/sorter.cpp:
##########
@@ -17,48 +17,154 @@
#include "vec/common/sort/sorter.h"
+#include "runtime/block_spill_manager.h"
#include "runtime/thread_context.h"
namespace doris::vectorized {
-void MergeSorterState::build_merge_tree(SortDescription& sort_description) {
- for (const auto& block : sorted_blocks) {
- cursors.emplace_back(block, sort_description);
+// When doing spillable sorting, each sorted block is spilled into a single
file.
+//
+// In order to decrease memory pressure when merging
+// multiple spilled blocks into one bigger sorted block, only part
+// of each spilled blocks are read back into memory at a time.
+//
+// Currently the spilled blocks are splitted into small sub blocks,
+// each sub block is serialized in PBlock format and appended
+// to the spill file.
+//
+// This number specifies the maximum size of sub blocks
+static constexpr int BLOCK_SPILL_BATCH_BYTES = 8 * 1024 * 1024;
+
+Status MergeSorterState::add_sorted_block(Block& block) {
+ auto rows = block.rows();
+ if (0 == rows) {
+ return Status::OK();
+ }
+ if (0 == avg_row_bytes_) {
+ avg_row_bytes_ = block.bytes() / rows;
+ spill_block_batch_size_ = (BLOCK_SPILL_BATCH_BYTES + avg_row_bytes_ -
1) / avg_row_bytes_;
}
- if (sorted_blocks.size() > 1) {
- for (auto& cursor : cursors)
priority_queue.push(MergeSortCursor(&cursor));
+ auto bytes_used = data_size();
+ if (is_spilled_ ||
Review Comment:
If spill to disk is triggered, the sort profile should have a element to
indicate this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
