Fokko commented on code in PR #2293:
URL: https://github.com/apache/iceberg-python/pull/2293#discussion_r2257966076
##########
pyiceberg/io/pyarrow.py:
##########
@@ -1401,42 +1400,24 @@ def _field_id(self, field: pa.Field) -> int:
def _get_column_projection_values(
- file: DataFile, projected_schema: Schema, partition_spec:
Optional[PartitionSpec], file_project_field_ids: Set[int]
-) -> Tuple[bool, Dict[str, Any]]:
+ file: DataFile, projected_schema: Schema, partition_spec: PartitionSpec,
file_project_field_ids: Set[int]
+) -> Dict[int, Any]:
"""Apply Column Projection rules to File Schema."""
project_schema_diff =
projected_schema.field_ids.difference(file_project_field_ids)
- should_project_columns = len(project_schema_diff) > 0
- projected_missing_fields: Dict[str, Any] = {}
+ if len(project_schema_diff) == 0:
+ return EMPTY_DICT
- if not should_project_columns:
- return False, {}
-
- partition_schema: StructType
- accessors: Dict[int, Accessor]
-
- if partition_spec is not None:
- partition_schema = partition_spec.partition_type(projected_schema)
- accessors = build_position_accessors(partition_schema)
- else:
- return False, {}
+ partition_schema = partition_spec.partition_type(projected_schema)
+ accessors = build_position_accessors(partition_schema)
+ projected_missing_fields = {}
for field_id in project_schema_diff:
for partition_field in partition_spec.fields_by_source_id(field_id):
if isinstance(partition_field.transform, IdentityTransform):
- accessor = accessors.get(partition_field.field_id)
-
- if accessor is None:
- continue
-
- # The partition field may not exist in the partition record of
the data file.
- # This can happen when new partition fields are introduced
after the file was written.
- try:
- if partition_value := accessor.get(file.partition):
- projected_missing_fields[partition_field.name] =
partition_value
- except IndexError:
- continue
+ if partition_value :=
accessors[partition_field.field_id].get(file.partition):
+ projected_missing_fields[field_id] = partition_value
Review Comment:
I think it makes sense to fail here, rather than suppress the Error. In the
case of an `IndexError` I think your table is corrupt.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
