kevinjqliu commented on code in PR #2293:
URL: https://github.com/apache/iceberg-python/pull/2293#discussion_r2258601170
##########
pyiceberg/io/pyarrow.py:
##########
@@ -1401,42 +1400,24 @@ def _field_id(self, field: pa.Field) -> int:
def _get_column_projection_values(
- file: DataFile, projected_schema: Schema, partition_spec:
Optional[PartitionSpec], file_project_field_ids: Set[int]
-) -> Tuple[bool, Dict[str, Any]]:
+ file: DataFile, projected_schema: Schema, partition_spec: PartitionSpec,
file_project_field_ids: Set[int]
+) -> Dict[int, Any]:
"""Apply Column Projection rules to File Schema."""
project_schema_diff =
projected_schema.field_ids.difference(file_project_field_ids)
- should_project_columns = len(project_schema_diff) > 0
- projected_missing_fields: Dict[str, Any] = {}
+ if len(project_schema_diff) == 0:
+ return EMPTY_DICT
- if not should_project_columns:
- return False, {}
-
- partition_schema: StructType
- accessors: Dict[int, Accessor]
-
- if partition_spec is not None:
- partition_schema = partition_spec.partition_type(projected_schema)
- accessors = build_position_accessors(partition_schema)
- else:
- return False, {}
+ partition_schema = partition_spec.partition_type(projected_schema)
+ accessors = build_position_accessors(partition_schema)
+ projected_missing_fields = {}
for field_id in project_schema_diff:
for partition_field in partition_spec.fields_by_source_id(field_id):
if isinstance(partition_field.transform, IdentityTransform):
- accessor = accessors.get(partition_field.field_id)
-
- if accessor is None:
- continue
-
- # The partition field may not exist in the partition record of
the data file.
- # This can happen when new partition fields are introduced
after the file was written.
- try:
- if partition_value := accessor.get(file.partition):
- projected_missing_fields[partition_field.name] =
partition_value
- except IndexError:
- continue
+ if partition_value :=
accessors[partition_field.field_id].get(file.partition):
+ projected_missing_fields[field_id] = partition_value
Review Comment:
good catch, i see its fixed in
https://github.com/apache/iceberg-python/pull/2293/files#diff-8d5e63f2a87ead8cebe2fd8ac5dcf2198d229f01e16bb9e06e21f7277c328abdR1672
now im worried about all the other places we use `.spec()`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
