geruh commented on code in PR #1626:
URL: https://github.com/apache/iceberg-python/pull/1626#discussion_r1960691377
##########
pyiceberg/table/inspect.py:
##########
@@ -657,3 +669,35 @@ def all_manifests(self) -> "pa.Table":
lambda args: self._generate_manifests_table(*args), [(snapshot,
True) for snapshot in snapshots]
)
return pa.concat_tables(manifests_by_snapshots)
+
+ def _all_files(self, data_file_filter: Optional[Set[DataFileContent]] =
None) -> "pa.Table":
+ import pyarrow as pa
+
+ snapshots = self.tbl.snapshots()
+ if not snapshots:
+ return pa.Table.from_pylist([], schema=self._get_files_schema())
+
+ executor = ExecutorFactory.get_or_create()
+ all_manifest_files_by_snapshot: Iterator[List[ManifestFile]] =
executor.map(
+ lambda args: args[0].manifests(self.tbl.io), [(snapshot,) for
snapshot in snapshots]
+ )
+ all_manifest_files = list(
+ {(manifest.manifest_path, manifest) for manifest_list in
all_manifest_files_by_snapshot for manifest in manifest_list}
+ )
+ all_files_by_manifest: Iterator[List[Dict[str, Any]]] = executor.map(
+ lambda args: self._files_by_manifest(*args), [(manifest,
data_file_filter) for _, manifest in all_manifest_files]
+ )
+ all_files_list = [file for files in all_files_by_manifest for file in
files]
+ return pa.Table.from_pylist(
+ all_files_list,
+ schema=self._get_files_schema(),
+ )
Review Comment:
I agree with this, the impl of the `_files_by_manifest` enforces uniqueness
which wasn't clear
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
