xanderbailey commented on code in PR #3624:
URL: https://github.com/apache/iceberg-python/pull/3624#discussion_r4584960011


##########
pyiceberg/table/update/snapshot.py:
##########
@@ -334,21 +334,22 @@ def _commit(self) -> UpdatesAndRequirements:
         manifest_list_file_path = 
location_provider.new_metadata_location(file_name)
         self._written_manifest_lists.append(manifest_list_file_path)
 
+        first_row_id: int | None = None
+
+        if self._transaction.table_metadata.format_version >= 3:
+            first_row_id = self._transaction.table_metadata.next_row_id
+
         with write_manifest_list(
             format_version=self._transaction.table_metadata.format_version,
             output_file=self._io.new_output(manifest_list_file_path),
             snapshot_id=self._snapshot_id,
             parent_snapshot_id=self._parent_snapshot_id,
             sequence_number=next_sequence_number,
             avro_compression=self._compression,
+            first_row_id=first_row_id,

Review Comment:
   I think we also need `added_rows` on the snapshot here, otherwise 
`AddSnapshotUpdate` fails on main with `Cannot add snapshot without added rows` 
(Java rejects it too). Might need to check that I'm reading this right. 



##########
pyiceberg/manifest.py:
##########
@@ -1330,18 +1373,102 @@ def prepare_entry(self, entry: ManifestEntry) -> 
ManifestEntry:
         return entry
 
 
+class ManifestWriterV3(ManifestWriterV2):
+    """Writes V3 manifest files.
+
+    The writer inherits the V2 sequence-number semantics; the V3 manifest entry
+    schema additionally carries `first_row_id`, `referenced_data_file`,
+    `content_offset` and `content_size_in_bytes` on the data file struct.
+
+    An optional `first_row_id` can be provided when rewriting a manifest whose
+    `first_row_id` is already known; it is carried into the produced manifest 
file
+    so the manifest list writer preserves it instead of assigning a new one. 
For
+    new manifests it is None and assigned when writing the manifest list.
+    """
+
+    _first_row_id: int | None
+
+    def __init__(
+        self,
+        spec: PartitionSpec,
+        schema: Schema,
+        output_file: OutputFile,
+        snapshot_id: int,
+        avro_compression: AvroCompressionCodec,
+        first_row_id: int | None = None,
+    ):
+        super().__init__(spec, schema, output_file, snapshot_id, 
avro_compression)
+        self._first_row_id = first_row_id
+
+    @property
+    def version(self) -> TableVersion:
+        return 3
+
+    def to_manifest_file(self) -> ManifestFile:

Review Comment:
   `super().to_manifest_file()` is already V3 layout so I think this can just 
be `manifest_file.first_row_id = self._first_row_id`?



##########
tests/table/test_snapshots.py:
##########
@@ -821,3 +821,40 @@ def 
test_overwrite_rejects_explicit_delete_without_parent_snapshot(
         with empty.transaction() as tx:
             with tx.update_snapshot().overwrite() as overwrite:
                 overwrite.delete_data_file(stale_file)
+
+
+def test_manifest_list_receives_first_row_id_for_v3(table_v3: Table) -> None:

Review Comment:
   nit: would be nice to do a real `_commit()` here rather than mocking, it 
would have caught the missing `added_rows` too.



##########
pyiceberg/manifest.py:
##########
@@ -1465,19 +1592,101 @@ def prepare_manifest(self, manifest_file: 
ManifestFile) -> ManifestFile:
         return wrapped_manifest_file
 
 
+class ManifestListWriterV3(ManifestListWriterV2):
+    """Writes V3 manifest lists, assigning `first_row_id` to data manifests.
+
+    Follows the spec's First Row ID Assignment: existing `first_row_id` values 
are
+    preserved, delete manifests are never assigned one, and data manifests 
without a
+    `first_row_id` are assigned a running value starting at the snapshot's
+    `first-row-id`, advanced by each assigned manifest's existing and added 
row counts.
+    """
+
+    _next_row_id: int
+
+    def __init__(
+        self,
+        output_file: OutputFile,
+        snapshot_id: int,
+        parent_snapshot_id: int | None,
+        sequence_number: int,
+        compression: AvroCompressionCodec,
+        first_row_id: int,
+    ):
+        super().__init__(output_file, snapshot_id, parent_snapshot_id, 
sequence_number, compression)
+        self._format_version = 3
+        self._meta = {
+            **self._meta,
+            "first-row-id": str(first_row_id),
+            "format-version": "3",
+        }
+        self._next_row_id = first_row_id
+
+    def __enter__(self) -> ManifestListWriter:
+        """Open the writer for writing, using the V3 record layout so 
`first_row_id` is written."""
+        self._writer = AvroOutputFile[ManifestFile](
+            output_file=self._output_file,
+            record_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
+            file_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
+            schema_name="manifest_file",
+            metadata=self._meta,
+        )
+        self._writer.__enter__()
+        return self
+
+    @property
+    def next_row_id(self) -> int:
+        """The row ID after the last assigned one; the table's `next-row-id` 
after this snapshot."""
+        return self._next_row_id
+
+    def _wrap(self, manifest_file: ManifestFile) -> ManifestFile:

Review Comment:
   Do we still need the field count detection? Nothing binds to V1/V2 anymore 
as far as I can see. Rust keeps one `ManifestFile` and converts when 
serializing. Could this just be a `copy()`? WDYT?



##########
pyiceberg/manifest.py:
##########
@@ -546,19 +547,23 @@ def sort_order_id(self) -> int | None:
 
     @property
     def first_row_id(self) -> int | None:
-        return self._data[16]
+        # V3-only field; absent on records bound to an older layout
+        return self._data[16] if len(self._data) > 16 else None

Review Comment:
   Claude finding, have not verified: This doesn't quite work for V2 records, 
V2 has `referenced_data_file` at index 16 so `first_row_id` returns the 
referenced file path. Not new to this PR, but since everything defaults to V3 
now I'd probably just drop the guards.



##########
pyiceberg/manifest.py:
##########
@@ -1465,19 +1592,101 @@ def prepare_manifest(self, manifest_file: 
ManifestFile) -> ManifestFile:
         return wrapped_manifest_file
 
 
+class ManifestListWriterV3(ManifestListWriterV2):
+    """Writes V3 manifest lists, assigning `first_row_id` to data manifests.
+
+    Follows the spec's First Row ID Assignment: existing `first_row_id` values 
are
+    preserved, delete manifests are never assigned one, and data manifests 
without a
+    `first_row_id` are assigned a running value starting at the snapshot's
+    `first-row-id`, advanced by each assigned manifest's existing and added 
row counts.
+    """
+
+    _next_row_id: int
+
+    def __init__(
+        self,
+        output_file: OutputFile,
+        snapshot_id: int,
+        parent_snapshot_id: int | None,
+        sequence_number: int,
+        compression: AvroCompressionCodec,
+        first_row_id: int,
+    ):
+        super().__init__(output_file, snapshot_id, parent_snapshot_id, 
sequence_number, compression)
+        self._format_version = 3
+        self._meta = {
+            **self._meta,
+            "first-row-id": str(first_row_id),
+            "format-version": "3",
+        }
+        self._next_row_id = first_row_id
+
+    def __enter__(self) -> ManifestListWriter:
+        """Open the writer for writing, using the V3 record layout so 
`first_row_id` is written."""
+        self._writer = AvroOutputFile[ManifestFile](
+            output_file=self._output_file,
+            record_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
+            file_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
+            schema_name="manifest_file",
+            metadata=self._meta,
+        )
+        self._writer.__enter__()
+        return self
+
+    @property
+    def next_row_id(self) -> int:
+        """The row ID after the last assigned one; the table's `next-row-id` 
after this snapshot."""
+        return self._next_row_id
+
+    def _wrap(self, manifest_file: ManifestFile) -> ManifestFile:
+        """Rebind a manifest file to the V3 record layout.
+
+        The manifest file's original layout version is recovered from its 
field count (each version's
+        layout has a distinct number of fields) so its positional data is 
zipped against the fields it
+        was actually bound to. Rebinding always produces a fresh record, so 
mutating the returned
+        manifest file (e.g. assigning `first_row_id`) never leaks back to the 
caller's record.
+        """
+        source_version = 
_layout_version_from_field_count(MANIFEST_LIST_FILE_SCHEMAS, 
len(manifest_file._data))
+        args = {
+            field.name: value
+            for field, value in 
zip(MANIFEST_LIST_FILE_SCHEMAS[source_version].fields, manifest_file._data, 
strict=True)
+        }
+        return ManifestFile.from_args(_table_format_version=3, **args)
+
+    def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
+        wrapped_manifest_file = 
super().prepare_manifest(self._wrap(manifest_file))
+
+        if wrapped_manifest_file.content == ManifestContent.DATA and 
wrapped_manifest_file.first_row_id is None:

Review Comment:
   Rust [clears `first_row_id` on delete 
manifests](https://github.com/apache/iceberg-rust/blob/eeb1e41f959589347ac98a0575a325cdc923f6e9/crates/iceberg/src/spec/manifest_list/writer.rs#L275-L278)
 and Java does the same, might be worth matching that here.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to