Gabriel39 commented on code in PR #66575:
URL: https://github.com/apache/doris/pull/66575#discussion_r3746568753
##########
be/src/format_v2/parquet/parquet_scan.cpp:
##########
@@ -252,6 +252,75 @@ std::vector<format::LocalColumnIndex>
physical_non_predicate_columns(
return columns;
}
+struct VariantRowGroupProjectionCounts {
+ size_t leaf = 0;
+ size_t full = 0;
+};
+
+VariantRowGroupProjectionCounts finalize_variant_projections_for_row_group(
+ const tparquet::RowGroup& row_group,
+ const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
+ std::vector<format::LocalColumnIndex>* projections) {
+ DORIS_CHECK(projections != nullptr);
+ VariantRowGroupProjectionCounts counts;
+ for (auto& projection : *projections) {
+ const int32_t local_id = projection.local_id();
+ if (local_id < 0 || local_id >=
static_cast<int32_t>(file_schema.size()) ||
+ !file_schema[local_id]->contains_variant) {
+ continue;
+ }
+ counts.leaf += detail::finalize_variant_leaf_projection_for_row_group(
+ row_group, *file_schema[local_id], &projection, &counts.full);
+ }
+ return counts;
+}
+
+void prepare_row_group_physical_projection(
+ const tparquet::RowGroup& row_group,
+ const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
+ const format::FileScanRequest& request, RowGroupReadPlan*
row_group_plan) {
+ DORIS_CHECK(row_group_plan != nullptr);
+ const auto may_project_variant_leaf = [&](const auto& projections) {
+ return std::ranges::any_of(projections, [&](const auto& projection) {
+ const int32_t local_id = projection.local_id();
+ return local_id >= 0 && local_id <
static_cast<int32_t>(file_schema.size()) &&
+ file_schema[local_id] != nullptr &&
file_schema[local_id]->contains_variant &&
+ format::is_partial_projection(&projection);
+ });
+ };
+ if (!may_project_variant_leaf(request.predicate_columns) &&
+ !may_project_variant_leaf(request.non_predicate_columns)) {
+ row_group_plan->physical_predicate_columns = {};
+ row_group_plan->physical_non_predicate_columns = {};
+ row_group_plan->has_row_group_physical_projection = false;
+ row_group_plan->variant_leaf_projection_columns = 0;
+ row_group_plan->variant_full_projection_columns = 0;
+ return;
+ }
+ row_group_plan->physical_predicate_columns = request.predicate_columns;
Review Comment:
Fixed in a5c2fa09598. Each row-group plan now stores only the ordinals that
require full Variant fallback; leaf-only groups retain no projection tree.
Metadata pruning consumes a transient physical leaf-id view, the physical
projection is materialized only for the current full-fallback row group, and
the reader/scheduler share one RowGroupScanPlan. The updated two-row-group test
asserts both compact full and empty leaf deltas.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
