Fokko commented on code in PR #1626:
URL: https://github.com/apache/iceberg-python/pull/1626#discussion_r2074993621
##########
pyiceberg/table/inspect.py:
##########
@@ -657,3 +671,30 @@ def all_manifests(self) -> "pa.Table":
lambda args: self._generate_manifests_table(*args), [(snapshot,
True) for snapshot in snapshots]
)
return pa.concat_tables(manifests_by_snapshots)
+
+ def _all_files(self, data_file_filter: Optional[Set[DataFileContent]] =
None) -> "pa.Table":
+ import pyarrow as pa
+
+ snapshots = self.tbl.snapshots()
+ if not snapshots:
+ return pa.Table.from_pylist([], schema=self._get_files_schema())
+
+ executor = ExecutorFactory.get_or_create()
+ manifest_lists = executor.map(lambda snapshot:
snapshot.manifests(self.tbl.io), snapshots)
+
+ unique_manifests = {(manifest.manifest_path, manifest) for
manifest_list in manifest_lists for manifest in manifest_list}
+
+ file_lists = executor.map(
+ lambda args: self._get_files_from_manifest(*args), [(manifest,
data_file_filter) for _, manifest in unique_manifests]
+ )
+
+ return pa.concat_tables(file_lists)
+
+ def all_files(self) -> "pa.Table":
+ return self._all_files()
+
+ def all_data_files(self) -> "pa.Table":
+ return self._all_files({DataFileContent.DATA})
+
+ def all_delete_files(self) -> "pa.Table":
+ return self._all_files({DataFileContent.POSITION_DELETES,
DataFileContent.EQUALITY_DELETES})
Review Comment:
This should also include Puffin files:
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
