geruh commented on code in PR #3285:
URL: https://github.com/apache/iceberg-python/pull/3285#discussion_r4547116911


##########
pyiceberg/table/delete_file_index.py:
##########
@@ -103,26 +171,33 @@ def _partition_key(spec_id: int, partition: Record | 
None) -> tuple[int, Record]
 
 
 class DeleteFileIndex:
-    """Indexes position delete files by partition and by exact data file 
path."""
+    """Indexes position and equality delete files by partition and by exact 
data file path."""
 
-    def __init__(self) -> None:
+    def __init__(self, schema: Schema | None = None) -> None:
+        self._schema = schema
         self._by_partition: dict[tuple[int, Record], PositionDeletes] = {}
         self._by_path: dict[str, PositionDeletes] = {}
+        self._eq_deletes: dict[tuple[int, Record] | None, EqualityDeletes] = {}
 
     def is_empty(self) -> bool:
-        return not self._by_partition and not self._by_path
+        return not self._by_partition and not self._by_path and not 
self._eq_deletes
 
     def add_delete_file(self, manifest_entry: ManifestEntry, partition_key: 
Record | None = None) -> None:
         delete_file = manifest_entry.data_file
         seq = manifest_entry.sequence_number or INITIAL_SEQUENCE_NUMBER
-        target_path = delete_file.referenced_data_file or 
_referenced_data_file_path(delete_file)
 
-        if target_path:
-            deletes = self._by_path.setdefault(target_path, PositionDeletes())
-            deletes.add(delete_file, seq)
-        else:
-            key = _partition_key(delete_file.spec_id or 0, partition_key)
-            deletes = self._by_partition.setdefault(key, PositionDeletes())
+        if delete_file.content == DataFileContent.POSITION_DELETES:
+            target_path = delete_file.referenced_data_file or 
_referenced_data_file_path(delete_file)
+            if target_path:
+                deletes = self._by_path.setdefault(target_path, 
PositionDeletes())
+                deletes.add(delete_file, seq)
+            else:
+                key = _partition_key(delete_file.spec_id or 0, partition_key)
+                deletes = self._by_partition.setdefault(key, PositionDeletes())
+                deletes.add(delete_file, seq)
+        elif delete_file.content == DataFileContent.EQUALITY_DELETES:
+            eq_key = _partition_key(delete_file.spec_id or 0, partition_key) 
if partition_key else None

Review Comment:
   nit: can we determine if partitioned from metadata instead of 
`partition_key` seems a bit safer.



##########
pyiceberg/io/pyarrow.py:
##########
@@ -1830,6 +1830,8 @@ def _read_all_delete_files(io: FileIO, tasks: 
Iterable[FileScanTask]) -> dict[st
     unique_deletes: dict[_DeleteFileKey, DataFile] = {}
     for task in tasks:
         for delete_file in task.delete_files:
+            if delete_file.content != DataFileContent.POSITION_DELETES:

Review Comment:
   I don't think we should silently skip equality deletes thats an invalid 
table state?



##########
pyiceberg/table/update/validate.py:
##########


Review Comment:
   looks like we also need to account for concurrent equality deletes from 
other engines here. We're using the head sequence instead of the operations 
starting sequence.
   
   Java: 
https://github.com/apache/iceberg/blob/e31ea5f39b3e7d4a536cad3a4dbdc16dcf1e1553/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java#L527-L531



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to