smaheshwar-pltr commented on code in PR #3364:
URL: https://github.com/apache/iceberg-python/pull/3364#discussion_r3260042251
##########
pyiceberg/table/__init__.py:
##########
@@ -1707,20 +1771,163 @@ def __init__(
row_filter: str | BooleanExpression = ALWAYS_TRUE,
selected_fields: tuple[str, ...] = ("*",),
case_sensitive: bool = True,
- snapshot_id: int | None = None,
options: Properties = EMPTY_DICT,
limit: int | None = None,
- catalog: Catalog | None = None,
- table_identifier: Identifier | None = None,
):
self.table_metadata = table_metadata
self.io = io
self.row_filter = _parse_row_filter(row_filter)
self.selected_fields = selected_fields
self.case_sensitive = case_sensitive
- self.snapshot_id = snapshot_id
self.options = options
self.limit = limit
+
+ @abstractmethod
+ def projection(self) -> Schema: ...
+
+ @abstractmethod
+ def plan_files(self) -> Iterable[ScanTask]: ...
+
+ def to_arrow(self) -> pa.Table:
+ """Read an Arrow table eagerly from this scan.
+
+ All rows will be loaded into memory at once.
+
+ Returns:
+ pa.Table: Materialized Arrow Table from the Iceberg table scan.
+ """
+ from pyiceberg.io.pyarrow import ArrowScan
+
+ return ArrowScan(
+ self.table_metadata, self.io, self.projection(), self.row_filter,
self.case_sensitive, self.limit
+ ).to_table(self.plan_files())
+
+ def to_arrow_batch_reader(self) -> pa.RecordBatchReader:
+ """Return an Arrow RecordBatchReader from this scan.
+
+ For large results, using a RecordBatchReader requires less memory than
+ loading an Arrow Table for the same scan, because a RecordBatch is read
+ one at a time.
+
+ Returns:
+ pa.RecordBatchReader: Arrow RecordBatchReader from the Iceberg
table scan,
+ which can be used to read a stream of record batches one by one.
+ """
+ import pyarrow as pa
+
+ from pyiceberg.io.pyarrow import ArrowScan, schema_to_pyarrow
+
+ target_schema = schema_to_pyarrow(self.projection())
+ batches = ArrowScan(
+ self.table_metadata, self.io, self.projection(), self.row_filter,
self.case_sensitive, self.limit
+ ).to_record_batches(self.plan_files())
+
+ return pa.RecordBatchReader.from_batches(
+ target_schema,
+ batches,
+ ).cast(target_schema)
+
+ def update(self: A, **overrides: Any) -> A:
+ """Create a copy of this table scan with updated fields."""
+ from inspect import signature
+
+ # Extract those attributes that are constructor parameters. We don't
use self.__dict__ as the kwargs to the
+ # constructors because it may contain additional attributes that are
not part of the constructor signature.
+ params = signature(type(self).__init__).parameters.keys() - {"self"}
# Skip "self" parameter
+ kwargs = {param: getattr(self, param) for param in params} # Assume
parameters are attributes
+
+ return type(self)(**{**kwargs, **overrides})
+
+ def select(self: A, *field_names: str) -> A:
+ if "*" in self.selected_fields:
+ return self.update(selected_fields=field_names)
+ return
self.update(selected_fields=tuple(set(self.selected_fields).intersection(set(field_names))))
+
+ def filter(self: A, expr: str | BooleanExpression) -> A:
+ return self.update(row_filter=And(self.row_filter,
_parse_row_filter(expr)))
+
+ def with_case_sensitive(self: A, case_sensitive: bool = True) -> A:
+ return self.update(case_sensitive=case_sensitive)
+
+ def to_pandas(self, **kwargs: Any) -> pd.DataFrame:
+ """Read a Pandas DataFrame eagerly from this Iceberg table scan.
+
+ Returns:
+ pd.DataFrame: Materialized Pandas Dataframe from the Iceberg table
scan
+ """
+ return self.to_arrow().to_pandas(**kwargs)
+
+ def to_duckdb(self, table_name: str, connection: DuckDBPyConnection | None
= None) -> DuckDBPyConnection:
+ """Shorthand for loading this table scan in DuckDB.
+
+ Returns:
+ DuckDBPyConnection: In memory DuckDB connection with the Iceberg
table scan.
+ """
+ import duckdb
+
+ con = connection or duckdb.connect(database=":memory:")
+ con.register(table_name, self.to_arrow())
+
+ return con
+
+ def to_ray(self) -> ray.data.dataset.Dataset:
+ """Read a Ray Dataset eagerly from this Iceberg table scan.
+
+ Returns:
+ ray.data.dataset.Dataset: Materialized Ray Dataset from the
Iceberg table scan
+ """
+ import ray
+
+ return ray.data.from_arrow(self.to_arrow())
+
+ def to_polars(self) -> pl.DataFrame:
+ """Read a Polars DataFrame from this Iceberg table scan.
+
+ Returns:
+ pl.DataFrame: Materialized Polars Dataframe from the Iceberg table
scan
+ """
+ import polars as pl
+
+ result = pl.from_arrow(self.to_arrow())
+ if isinstance(result, pl.Series):
+ result = result.to_frame()
+
+ return result
+
+
+S = TypeVar("S", bound="TableScan", covariant=True)
+
+
+class TableScan(BaseScan, ABC):
Review Comment:
Was a direct `ABC`; now extends `BaseScan`. All previously-present fields,
methods, and abstract API are preserved (see
https://github.com/apache/iceberg-python/pull/3364#discussion_r3255000453). The
only behavioural delta is that previously-abstract methods on `TableScan`
(`to_pandas`, `to_polars`) now have default implementations inherited from
`BaseScan`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
