soumya-ghosh commented on code in PR #1626:
URL: https://github.com/apache/iceberg-python/pull/1626#discussion_r2078557847
##########
pyiceberg/table/inspect.py:
##########
@@ -657,3 +671,30 @@ def all_manifests(self) -> "pa.Table":
lambda args: self._generate_manifests_table(*args), [(snapshot,
True) for snapshot in snapshots]
)
return pa.concat_tables(manifests_by_snapshots)
+
+ def _all_files(self, data_file_filter: Optional[Set[DataFileContent]] =
None) -> "pa.Table":
+ import pyarrow as pa
+
+ snapshots = self.tbl.snapshots()
+ if not snapshots:
+ return pa.Table.from_pylist([], schema=self._get_files_schema())
+
+ executor = ExecutorFactory.get_or_create()
+ manifest_lists = executor.map(lambda snapshot:
snapshot.manifests(self.tbl.io), snapshots)
+
+ unique_manifests = {(manifest.manifest_path, manifest) for
manifest_list in manifest_lists for manifest in manifest_list}
+
+ file_lists = executor.map(
+ lambda args: self._get_files_from_manifest(*args), [(manifest,
data_file_filter) for _, manifest in unique_manifests]
+ )
+
+ return pa.concat_tables(file_lists)
+
+ def all_files(self) -> "pa.Table":
+ return self._all_files()
+
+ def all_data_files(self) -> "pa.Table":
+ return self._all_files({DataFileContent.DATA})
+
+ def all_delete_files(self) -> "pa.Table":
+ return self._all_files({DataFileContent.POSITION_DELETES,
DataFileContent.EQUALITY_DELETES})
Review Comment:
@Fokko Added an integration test for table with format version 3, used Spark
to write through pyiceberg to V3 table were failing.
Note that, the outputs of files metadata (and all other related tables) do
not completely match with Spark counterparts due to additional columns in like
`first_row_id`, `referenced_data_file`, `content_offset`,
`content_size_in_bytes`. This needs to added first in DataFile class then
propagated as required. Should be addressed in different issue, will it part of
V3 tracking issue?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
