geruh commented on code in PR #3285:
URL: https://github.com/apache/iceberg-python/pull/3285#discussion_r4547116911
##########
pyiceberg/table/delete_file_index.py:
##########
@@ -103,26 +171,33 @@ def _partition_key(spec_id: int, partition: Record |
None) -> tuple[int, Record]
class DeleteFileIndex:
- """Indexes position delete files by partition and by exact data file
path."""
+ """Indexes position and equality delete files by partition and by exact
data file path."""
- def __init__(self) -> None:
+ def __init__(self, schema: Schema | None = None) -> None:
+ self._schema = schema
self._by_partition: dict[tuple[int, Record], PositionDeletes] = {}
self._by_path: dict[str, PositionDeletes] = {}
+ self._eq_deletes: dict[tuple[int, Record] | None, EqualityDeletes] = {}
def is_empty(self) -> bool:
- return not self._by_partition and not self._by_path
+ return not self._by_partition and not self._by_path and not
self._eq_deletes
def add_delete_file(self, manifest_entry: ManifestEntry, partition_key:
Record | None = None) -> None:
delete_file = manifest_entry.data_file
seq = manifest_entry.sequence_number or INITIAL_SEQUENCE_NUMBER
- target_path = delete_file.referenced_data_file or
_referenced_data_file_path(delete_file)
- if target_path:
- deletes = self._by_path.setdefault(target_path, PositionDeletes())
- deletes.add(delete_file, seq)
- else:
- key = _partition_key(delete_file.spec_id or 0, partition_key)
- deletes = self._by_partition.setdefault(key, PositionDeletes())
+ if delete_file.content == DataFileContent.POSITION_DELETES:
+ target_path = delete_file.referenced_data_file or
_referenced_data_file_path(delete_file)
+ if target_path:
+ deletes = self._by_path.setdefault(target_path,
PositionDeletes())
+ deletes.add(delete_file, seq)
+ else:
+ key = _partition_key(delete_file.spec_id or 0, partition_key)
+ deletes = self._by_partition.setdefault(key, PositionDeletes())
+ deletes.add(delete_file, seq)
+ elif delete_file.content == DataFileContent.EQUALITY_DELETES:
+ eq_key = _partition_key(delete_file.spec_id or 0, partition_key)
if partition_key else None
Review Comment:
nit: can we determine if partitioned from metadata instead of
`partition_key` seems a bit safer.
##########
pyiceberg/io/pyarrow.py:
##########
@@ -1830,6 +1830,8 @@ def _read_all_delete_files(io: FileIO, tasks:
Iterable[FileScanTask]) -> dict[st
unique_deletes: dict[_DeleteFileKey, DataFile] = {}
for task in tasks:
for delete_file in task.delete_files:
+ if delete_file.content != DataFileContent.POSITION_DELETES:
Review Comment:
I don't think we should silently skip equality deletes thats an invalid
table state?
##########
pyiceberg/table/update/validate.py:
##########
Review Comment:
looks like we also need to account for concurrent equality deletes from
other engines here. We're using the head sequence instead of the operations
starting sequence.
Java:
https://github.com/apache/iceberg/blob/e31ea5f39b3e7d4a536cad3a4dbdc16dcf1e1553/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java#L527-L531
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]