yiguolei commented on code in PR #66472:
URL: https://github.com/apache/doris/pull/66472#discussion_r3765575881
##########
be/src/storage/segment/segment_iterator.cpp:
##########
@@ -2175,75 +1981,57 @@ bool SegmentIterator::_prune_column(ColumnId cid,
MutableColumnPtr& column,
}
bool SegmentIterator::_can_skip_reading_extra_column(ColumnId cid) {
- if (!_opts.extra_columns.contains(cid) || _is_pred_column.empty()) {
+ if (!_opts.extra_columns.contains(cid)) {
return false;
}
- DCHECK_EQ(_is_pred_column.size(), _is_common_expr_column.size());
- DCHECK_LT(cid, _is_pred_column.size());
// extra_columns is only an optimization hint. The real value is still
// required when the column participates in expression materialization or
// any predicate path.
- return !_virtual_column_exprs.contains(cid) && !_has_delete_predicate(cid)
&&
- !_is_pred_column[cid] && !_is_common_expr_column[cid];
-}
-
-Status SegmentIterator::_read_columns(const std::vector<ColumnId>& column_ids,
- MutableColumns& column_block, size_t
nrows) {
- for (auto cid : column_ids) {
- auto& column = column_block[cid];
- size_t rows_read = nrows;
- if (_prune_column(cid, column, rows_read)) {
- continue;
- }
- RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read,
column));
- if (nrows != rows_read) {
- return Status::Error<ErrorCode::INTERNAL_ERROR>("nrows({}) !=
rows_read({})", nrows,
- rows_read);
- }
- }
- return Status::OK();
+ return !_virtual_column_exprs.contains(cid) &&
!_column_states[cid].is_predicate() &&
+ !_column_states[cid].is_common_expr;
}
Status SegmentIterator::_init_current_block(Block* block,
std::vector<MutableColumnPtr>&
current_columns,
uint32_t nrows_read_limit) {
- block->clear_column_data(_schema->num_column_ids());
+ block->clear_column_data(cast_set<int64_t>(_schema->num_block_columns()));
- for (size_t i = 0; i < _schema->num_column_ids(); i++) {
- auto cid = _schema->column_id(i);
- const auto* column_desc = _schema->column(cid);
+ for (ColumnId i = 0; i < _schema->num_read_columns(); i++) {
+ if (!_is_active_read_column(i)) {
+ continue;
+ }
+ const auto* column_desc = _schema->column(i);
+
+ const auto& file_column_type = _storage_name_and_type[i].second;
+ const auto& expected_type = _schema->data_type(i);
+ if (_column_states[i].is_predicate()) {
+ if (current_columns[i].get() == nullptr) {
+ return Status::InternalError("SegmentIterator meet invalid
column, id={}, name={}",
+ i, column_desc->name());
+ }
+ current_columns[i]->clear();
+ continue;
+ }
- auto file_column_type = _storage_name_and_type[cid].second;
- auto expected_type = Schema::get_data_type_ptr(*column_desc);
- if (!_is_pred_column[cid] &&
!file_column_type->equals(*expected_type)) {
- // The storage layer type is different from schema needed type, so
we use storage
- // type to read columns instead of schema type for safety
+ DCHECK_LT(i, _schema->num_block_columns());
+ if (!file_column_type->equals(*expected_type)) {
+ // The column iterator writes a different type from the
read-schema type, so
+ // materialize into an intermediate column and convert it after
reading.
VLOG_DEBUG << fmt::format(
- "Recreate column with expected type {}, file column type
{}, col_name {}, "
+ "Recreate column with expected type {}, materialization
type {}, col_name {}, "
"col_path {}",
block->get_by_position(i).type->get_name(),
file_column_type->get_name(),
column_desc->name(),
column_desc->path_info_ptr() == nullptr
? ""
: column_desc->path_info_ptr()->get_path());
// TODO reuse
- current_columns[cid] = file_column_type->create_column();
- current_columns[cid]->reserve(nrows_read_limit);
+ current_columns[i] = file_column_type->create_column();
+ current_columns[i]->reserve(nrows_read_limit);
} else {
- // the column in block must clear() here to insert new data
- if (_is_pred_column[cid] ||
- i >= block->columns()) { //todo(wb) maybe we can release it
after output block
- if (current_columns[cid].get() == nullptr) {
- return Status::InternalError(
- "SegmentIterator meet invalid column, id={},
name={}", cid,
- _schema->column(cid)->name());
- }
- current_columns[cid]->clear();
- } else { // non-predicate column
- current_columns[cid] =
std::move(*block->get_by_position(i).column).mutate();
- current_columns[cid]->reserve(nrows_read_limit);
- }
+ current_columns[i] =
std::move(*block->get_by_position(i).column).mutate();
Review Comment:
这里check 一下，这个block 里的column 的类型，和 它必须是empty
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
