kevinjqliu commented on code in PR #2265: URL: https://github.com/apache/iceberg-python/pull/2265#discussion_r2249326961
########## tests/conftest.py: ########## @@ -1858,15 +1861,40 @@ def simple_map() -> MapType: @pytest.fixture(scope="session") -def generated_manifest_entry_file(avro_schema_manifest_entry: Dict[str, Any]) -> Generator[str, None, None]: +def test_schema() -> Schema: + return Schema( + NestedField(1, "VendorID", IntegerType(), False), NestedField(2, "tpep_pickup_datetime", TimestampType(), False) + ) + + [email protected](scope="session") +def test_partition_spec() -> Schema: + return PartitionSpec( + PartitionField(1, 1000, IdentityTransform(), "VendorID"), + PartitionField(2, 1001, DayTransform(), "tpep_pickup_day"), + ) + + [email protected](scope="session") +def generated_manifest_entry_file( + avro_schema_manifest_entry: Dict[str, Any], test_schema: Schema, test_partition_spec: PartitionSpec +) -> Generator[str, None, None]: from fastavro import parse_schema, writer parsed_schema = parse_schema(avro_schema_manifest_entry) with TemporaryDirectory() as tmpdir: tmp_avro_file = tmpdir + "/manifest.avro" with open(tmp_avro_file, "wb") as out: - writer(out, parsed_schema, manifest_entry_records) + writer( + out, + parsed_schema, + manifest_entry_records, Review Comment: here a script to repro. i built pyiceberg-core locally and ran `RUST_BACKTRACE=1 hatch env run -e dev python repro.py` heres the repro script: the files are generated from pyiceberg tests that i breakpointed ``` from pyiceberg_core import manifest manifest_file_path = '/var/folders/wq/mxmvfjtd1kjcrm3bdfrn2zd80000gn/T/tmpm_zqzqi2/manifest.avro' with open(manifest_file_path, 'rb') as file: manifest_entries = manifest.read_manifest_entries(file.read()).entries() print("Manifest entries: len =", len(manifest_entries)) for entry in manifest_entries: print("status:", entry.status) print("snapshot_id:", entry.snapshot_id) print("sequence_number:", entry.sequence_number) print("file_sequence_number:", entry.file_sequence_number) data_file = entry.data_file print("data_file:", data_file) # See note below for expanding this print() from pyiceberg_core import manifest manifest_list_file_path = '/var/folders/wq/mxmvfjtd1kjcrm3bdfrn2zd80000gn/T/tmpltncyujb/manifest.avro' with open(manifest_list_file_path, 'rb') as file: manifest_file_entries = manifest.read_manifest_list(file.read()).entries() print("Manifest entries: len =", len(manifest_file_entries)) for entry in manifest_file_entries: print("manifest_path:", entry.manifest_path) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
