Baunsgaard commented on code in PR #3658:
URL: https://github.com/apache/iceberg-rust/pull/3658#discussion_r4550408704
##########
crates/iceberg/src/arrow/reader/projection.rs:
##########
@@ -177,45 +179,44 @@ impl ArrowReader {
type_promotion_is_valid: fn(Option<&PrimitiveType>,
Option<&PrimitiveType>) -> bool,
) -> Result<ProjectionMask> {
let mut column_map = HashMap::with_capacity(leaf_field_ids.len());
- let fields = arrow_schema.fields();
// HashSet for O(1) membership checks instead of O(n) slice scans.
let leaf_field_id_set: HashSet<i32> =
leaf_field_ids.iter().copied().collect();
- // Pre-project only the fields that have been selected, possibly
avoiding converting
- // some Arrow types that are not yet supported.
- let projected_arrow_schema = ArrowSchema::new_with_metadata(
- fields.filter_leaves(|_, f| {
- parquet_field_id(f).is_some_and(|field_id|
leaf_field_id_set.contains(&field_id))
- }),
- arrow_schema.metadata().clone(),
- );
- let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?;
-
- fields.filter_leaves(|idx, field| {
- let Some(field_id) = parquet_field_id(field) else {
- return false;
- };
-
- let iceberg_field = iceberg_schema_of_task.field_by_id(field_id);
- let parquet_iceberg_field = iceberg_schema.field_by_id(field_id);
-
- if iceberg_field.is_none() || parquet_iceberg_field.is_none() {
- return false;
+ // Convert selected leaves one at a time, only converting the selected
Arrow types.
+ // Converting a pre-projected schema fails when a map keeps only its
key (#3557).
+ let mut selected_leaves = Vec::with_capacity(leaf_field_ids.len());
+ arrow_schema.fields().filter_leaves(|idx, field| {
+ if let Some(field_id) = parquet_field_id(field)
+ && leaf_field_id_set.contains(&field_id)
+ {
+ selected_leaves.push((idx, field_id, field.clone()));
}
+ false
+ });
+ // Must stay out of the map value rescue below, which would cast them
instead.
+ let mut type_rejected_leaves = HashSet::new();
+ for (idx, field_id, field) in selected_leaves {
+ let Some(iceberg_field) =
iceberg_schema_of_task.field_by_id(field_id) else {
+ continue;
+ };
+ let file_schema =
arrow_schema_to_schema(&ArrowSchema::new(vec![field]))?;
Review Comment:
Projected subtrees now go through the same duplicate-id check, so a
duplicate id inside a projected struct fails with DataInvalid instead of being
accepted. This is covered by
test_duplicate_field_id_in_projected_subtree_fails. Fixed in 865fe3cb2.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]