gavinchou commented on code in PR #63504:
URL: https://github.com/apache/doris/pull/63504#discussion_r3301924044
##########
be/src/io/cache/block_file_cache.cpp:
##########
@@ -2090,35 +2191,46 @@ void BlockFileCache::run_background_block_lru_update() {
Thread::set_self_name("run_background_block_lru_update");
std::vector<FileBlockSPtr> batch;
while (!_close) {
- int64_t interval_ms =
config::file_cache_background_block_lru_update_interval_ms;
- size_t batch_limit =
- config::file_cache_background_block_lru_update_qps_limit *
interval_ms / 1000;
+ size_t backlog = _need_update_lru_blocks.size();
+ QueueConsumePlan plan = build_block_lru_update_plan(backlog);
{
std::unique_lock close_lock(_close_mtx);
- _close_cv.wait_for(close_lock,
std::chrono::milliseconds(interval_ms));
+ _close_cv.wait_for(close_lock,
std::chrono::milliseconds(plan.interval_ms));
if (_close) {
break;
}
}
batch.clear();
- batch.reserve(batch_limit);
- size_t drained = _need_update_lru_blocks.drain(batch_limit, &batch);
+ batch.reserve(plan.batch_limit);
+ size_t drained = _need_update_lru_blocks.drain(plan.batch_limit,
&batch);
if (drained == 0) {
*_need_update_lru_blocks_length_recorder <<
_need_update_lru_blocks.size();
continue;
}
int64_t duration_ns = 0;
- {
- SCOPED_CACHE_LOCK(_mutex, this);
- SCOPED_RAW_TIMER(&duration_ns);
- for (auto& block : batch) {
- update_block_lru(block, cache_lock);
+ const size_t slice_batch = positive_size_or_default(
+ config::file_cache_block_lru_update_lock_slice_batch, drained);
+ for (size_t begin = 0; begin < batch.size(); begin += slice_batch) {
+ const size_t end = std::min(begin + slice_batch, batch.size());
+ {
+ SCOPED_CACHE_LOCK(_mutex, this);
+ SCOPED_RAW_TIMER(&duration_ns);
+ for (size_t i = begin; i < end; ++i) {
+ update_block_lru(batch[i], cache_lock);
+ }
}
}
*_update_lru_blocks_latency_us << (duration_ns / 1000);
*_need_update_lru_blocks_length_recorder <<
_need_update_lru_blocks.size();
+ if (backlog >= static_cast<size_t>(std::max<int64_t>(
Review Comment:
what should we do when the consumption can not catch up?
logging does not resolve the memory issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
