Fokko commented on code in PR #1717:
URL: https://github.com/apache/iceberg-python/pull/1717#discussion_r1973893895
##########
pyiceberg/io/pyarrow.py:
##########
@@ -2464,38 +2464,37 @@ def _check_pyarrow_schema_compatible(
_check_schema_compatible(requested_schema, provided_schema)
-def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata,
file_paths: Iterator[str]) -> Iterator[DataFile]:
- for file_path in file_paths:
- input_file = io.new_input(file_path)
- with input_file.open() as input_stream:
- parquet_metadata = pq.read_metadata(input_stream)
-
- if visit_pyarrow(parquet_metadata.schema.to_arrow_schema(), _HasIds()):
- raise NotImplementedError(
- f"Cannot add file {file_path} because it has field IDs.
`add_files` only supports addition of files without field_ids"
- )
- schema = table_metadata.schema()
- _check_pyarrow_schema_compatible(schema,
parquet_metadata.schema.to_arrow_schema())
-
- statistics = data_file_statistics_from_parquet_metadata(
- parquet_metadata=parquet_metadata,
- stats_columns=compute_statistics_plan(schema,
table_metadata.properties),
- parquet_column_mapping=parquet_path_to_id_mapping(schema),
- )
- data_file = DataFile(
- content=DataFileContent.DATA,
- file_path=file_path,
- file_format=FileFormat.PARQUET,
- partition=statistics.partition(table_metadata.spec(),
table_metadata.schema()),
- file_size_in_bytes=len(input_file),
- sort_order_id=None,
- spec_id=table_metadata.default_spec_id,
- equality_ids=None,
- key_metadata=None,
- **statistics.to_serialized_dict(),
+def parquet_file_to_data_file(io: FileIO, table_metadata: TableMetadata,
file_path: str) -> DataFile:
+ input_file = io.new_input(file_path)
+ with input_file.open() as input_stream:
+ parquet_metadata = pq.read_metadata(input_stream)
+
+ if visit_pyarrow(parquet_metadata.schema.to_arrow_schema(), _HasIds()):
+ raise NotImplementedError(
+ f"Cannot add file {file_path} because it has field IDs.
`add_files` only supports addition of files without field_ids"
)
+ schema = table_metadata.schema()
+ _check_pyarrow_schema_compatible(schema,
parquet_metadata.schema.to_arrow_schema())
Review Comment:
`to_arrow_schema()` is called on lines 2483 and 2479. Would be good to call
this just once
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
