Fokko commented on code in PR #1717:
URL: https://github.com/apache/iceberg-python/pull/1717#discussion_r1970024778
##########
pyiceberg/io/pyarrow.py:
##########
@@ -2464,38 +2464,37 @@ def _check_pyarrow_schema_compatible(
_check_schema_compatible(requested_schema, provided_schema)
-def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata,
file_paths: Iterator[str]) -> Iterator[DataFile]:
- for file_path in file_paths:
- input_file = io.new_input(file_path)
- with input_file.open() as input_stream:
- parquet_metadata = pq.read_metadata(input_stream)
-
- if visit_pyarrow(parquet_metadata.schema.to_arrow_schema(), _HasIds()):
- raise NotImplementedError(
- f"Cannot add file {file_path} because it has field IDs.
`add_files` only supports addition of files without field_ids"
- )
- schema = table_metadata.schema()
- _check_pyarrow_schema_compatible(schema,
parquet_metadata.schema.to_arrow_schema())
-
- statistics = data_file_statistics_from_parquet_metadata(
- parquet_metadata=parquet_metadata,
- stats_columns=compute_statistics_plan(schema,
table_metadata.properties),
- parquet_column_mapping=parquet_path_to_id_mapping(schema),
- )
- data_file = DataFile(
- content=DataFileContent.DATA,
- file_path=file_path,
- file_format=FileFormat.PARQUET,
- partition=statistics.partition(table_metadata.spec(),
table_metadata.schema()),
- file_size_in_bytes=len(input_file),
- sort_order_id=None,
- spec_id=table_metadata.default_spec_id,
- equality_ids=None,
- key_metadata=None,
- **statistics.to_serialized_dict(),
+def parquet_file_to_data_file(io: FileIO, table_metadata: TableMetadata,
file_path: str) -> DataFile:
+ input_file = io.new_input(file_path)
+ with input_file.open() as input_stream:
+ parquet_metadata = pq.read_metadata(input_stream)
+
+ if visit_pyarrow(parquet_metadata.schema.to_arrow_schema(), _HasIds()):
+ raise NotImplementedError(
+ f"Cannot add file {file_path} because it has field IDs.
`add_files` only supports addition of files without field_ids"
)
+ schema = table_metadata.schema()
+ _check_pyarrow_schema_compatible(schema,
parquet_metadata.schema.to_arrow_schema())
Review Comment:
We're converting the schema multiple times. Since we're optimizing for
performance now, we probably want to store this in a variable to reduce GIL
congestion
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
