HappenLee commented on code in PR #15031: URL: https://github.com/apache/doris/pull/15031#discussion_r1051541079
########## be/src/pipeline/exec/data_queue.cpp: ########## @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "data_queue.h" + +#include <mutex> + +#include "vec/core/block.h" + +namespace doris { +namespace vectorized { +class Block; +} +namespace pipeline { + +DataQueue::DataQueue(int child_count) { + _child_count = child_count; + _flag_queue_idx = 0; + _queue_blocks.resize(child_count); + _free_blocks.resize(child_count); + _queue_blocks_lock.resize(child_count); + _free_blocks_lock.resize(child_count); + _is_finished.resize(child_count); + _is_canceled.resize(child_count); + _cur_bytes_in_queue.resize(child_count); + _cur_blocks_nums_in_queue.resize(child_count); + for (int i = 0; i < child_count; ++i) { + _queue_blocks_lock[i].reset(new std::mutex()); + _free_blocks_lock[i].reset(new std::mutex()); + _is_finished[i] = false; + _is_canceled[i] = false; + _cur_bytes_in_queue[i] = 0; + _cur_blocks_nums_in_queue[i] = 0; + } +} + +std::unique_ptr<vectorized::Block> DataQueue::get_free_block(int child_idx) { + { + std::lock_guard<std::mutex> l(*_free_blocks_lock[child_idx]); + if (!_free_blocks[child_idx].empty()) { + auto block = std::move(_free_blocks[child_idx].front()); + _free_blocks[child_idx].pop_front(); + return block; + } + } + + return std::make_unique<vectorized::Block>(); +} + +void DataQueue::push_free_block(std::unique_ptr<vectorized::Block> block, int child_idx) { + DCHECK(block->rows() == 0); + std::lock_guard<std::mutex> l(*_free_blocks_lock[child_idx]); + _free_blocks[child_idx].emplace_back(std::move(block)); +} + +//use sink to check can_write +bool DataQueue::has_enough_space_to_push(int child_idx) { + return _cur_bytes_in_queue[child_idx].load() < MAX_BYTE_OF_QUEUE / 2; +} + +//use source to check can_read +bool DataQueue::has_data_or_finished(int child_idx) { + return remaining_has_data() || _is_finished[child_idx]; +} + +//check which queue have data, and save the idx in _flag_queue_idx, +//so next loop, will check the record idx + 1 first +//maybe it's useful with many queue, others maybe always 0 +bool DataQueue::remaining_has_data() { + int count = _child_count - 1; + while (count >= 0) { + _flag_queue_idx = (_flag_queue_idx + 1) % _child_count; + if (_cur_blocks_nums_in_queue[_flag_queue_idx] > 0) { + return true; + } + count--; + } + return false; +} + +//the _flag_queue_idx indicate which queue has data, and in check can_read +//will be set idx in remaining_has_data function +Status DataQueue::get_block_from_queue(std::unique_ptr<vectorized::Block>* output_block, + int* child_idx) { + if (_is_canceled[_flag_queue_idx]) { + return Status::InternalError("AggContext canceled"); + } + + if (_cur_blocks_nums_in_queue[_flag_queue_idx] > 0) { Review Comment: danger !!! two thread call function will coredump -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
