zhiqiang-hhhh commented on code in PR #40144:
URL: https://github.com/apache/doris/pull/40144#discussion_r1745167162
##########
be/src/vec/exec/scan/scanner_scheduler.cpp:
##########
@@ -267,32 +271,42 @@ void
ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
if (free_block == nullptr) {
break;
}
+ // We got a new created block or a reused block.
+ ctx->update_peak_memory_usage(free_block->allocated_bytes());
+ ctx->update_peak_memory_usage(-free_block->allocated_bytes());
status = scanner->get_block_after_projects(state,
free_block.get(), &eos);
+ // Projection will truncate useless columns, makes block size
change.
+ auto free_block_bytes = free_block->allocated_bytes();
+ ctx->update_peak_memory_usage(free_block_bytes);
first_read = false;
if (!status.ok()) {
LOG(WARNING) << "Scan thread read VScanner failed: " <<
status.to_string();
break;
}
- auto free_block_bytes = free_block->allocated_bytes();
raw_bytes_read += free_block_bytes;
+
if (!scan_task->cached_blocks.empty() &&
scan_task->cached_blocks.back().first->rows() +
free_block->rows() <=
ctx->batch_size()) {
- size_t block_size =
scan_task->cached_blocks.back().first->allocated_bytes();
vectorized::MutableBlock mutable_block(
scan_task->cached_blocks.back().first.get());
+
ctx->update_peak_memory_usage(-mutable_block.allocated_bytes());
status = mutable_block.merge(*free_block);
+
ctx->update_peak_memory_usage(mutable_block.allocated_bytes());
if (!status.ok()) {
LOG(WARNING) << "Block merge failed: " <<
status.to_string();
break;
}
+ scan_task->cached_blocks.back().second =
mutable_block.allocated_bytes();
scan_task->cached_blocks.back().first.get()->set_columns(
std::move(mutable_block.mutable_columns()));
+ // If block can be reused, its memory usage will be added
back.
ctx->return_free_block(std::move(free_block));
- ctx->inc_free_block_usage(
-
scan_task->cached_blocks.back().first->allocated_bytes() - block_size);
+ // Return block succeed or not, this free_block is not
used by this scan task any more.
+ ctx->update_peak_memory_usage(-free_block_bytes);
Review Comment:
return free block 里没有处理这个，必须要在这里做才是对的。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
