This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit db5300fba463372295e14a0848733782f296207a
Author: Chenyang Sun <csun5...@gmail.com>
AuthorDate: Mon Jul 24 08:47:19 2023 +0800
fix(compaction) release the block and segment iterator after reading to the
end of the segment file (#22082)
When reading to the end of the segment file, clearing the block did not
release the memory, leading to high memory usage during compaction.
When reading through segment file for columns that are dictionary encoded,
the column iterator in the segment iterator will hold the dictionary. Release
the segment iterator to free up the dictionary.
---
be/src/vec/olap/vertical_merge_iterator.cpp | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/be/src/vec/olap/vertical_merge_iterator.cpp
b/be/src/vec/olap/vertical_merge_iterator.cpp
index 7285531d71..7ace9b457c 100644
--- a/be/src/vec/olap/vertical_merge_iterator.cpp
+++ b/be/src/vec/olap/vertical_merge_iterator.cpp
@@ -356,6 +356,13 @@ Status VerticalMergeIteratorContext::_load_next_block() {
if (!st.ok()) {
_valid = false;
if (st.is<END_OF_FILE>()) {
+ // When reading to the end of the segment file, clearing the
block did not release the memory.
+ // Directly releasing the block to free up memory.
+ _block.reset();
+ // When reading through segment file for columns that are
dictionary encoded,
+ // the column iterator in the segment iterator will hold the
dictionary.
+ // Release the segment iterator to free up the dictionary.
+ _iter.reset();
return Status::OK();
} else {
return st;
@@ -601,7 +608,9 @@ Status VerticalFifoMergeIterator::init(const
StorageReadOptions& opts) {
Status VerticalMaskMergeIterator::check_all_iter_finished() {
for (auto iter : _origin_iter_ctx) {
if (iter->inited()) {
- RETURN_IF_ERROR(iter->advance());
+ if (iter->valid()) {
+ RETURN_IF_ERROR(iter->advance());
+ }
DCHECK(!iter->valid());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
