Fokko commented on PR #7831: URL: https://github.com/apache/iceberg/pull/7831#issuecomment-1631183637
For some reason my big comment was collapsed: Since Arrow is column-oriented, I'm sure that it will follow the order of the write schema:  The reason I try to avoid using too many internal details from PyArrow is that we support PyArrow `[9.0.0, 12.0.1]` currently. There is no guarantee that all these internals stay the same, therefore I think we should do as much as possible within our own control (also regarding the internal Parquet types, or how the column names are structured). We already have the write schema, so we can easily filter out the primitive types: ```python class PyArrowStatisticsCollector(PreOrderSchemaVisitor[List[StatisticsCollector]]): _field_id = 0 _schema: Schema _properties: Properties def __init__(self, schema: Schema, properties: Dict[str, str]): self._schema = schema self._properties = properties def schema(self, schema: Schema, struct_result: Callable[[], List[StatisticsCollector]]) -> List[StatisticsCollector]: return struct_result() def struct(self, struct: StructType, field_results: List[Callable[[], List[StatisticsCollector]]]) -> List[StatisticsCollector]: return list(chain(*[result() for result in field_results])) def field(self, field: NestedField, field_result: Callable[[], List[StatisticsCollector]]) -> List[StatisticsCollector]: self._field_id = field.field_id result = field_result() return result def list(self, list_type: ListType, element_result: Callable[[], List[StatisticsCollector]]) -> List[StatisticsCollector]: self._field_id = list_type.element_id return element_result() def map(self, map_type: MapType, key_result: Callable[[], List[StatisticsCollector]], value_result: Callable[[], List[StatisticsCollector]]) -> List[StatisticsCollector]: self._field_id = map_type.key_id k = key_result() self._field_id = map_type.value_id v = value_result() return k + v def primitive(self, primitive: PrimitiveType) -> List[StatisticsCollector]: return [StatisticsCollector( field_id=self._field_id, iceberg_type=primitive, mode=MetricsMode.TRUNC # schema # self._properties.get(f"write.metadata.metrics.column.{schema.find_column_name(self._field_id)}") )] ``` This way we get a nice list of columns that we need to collect statistics for. We have: ```python @dataclass(frozen=True) class StatisticsCollector: field_id: int iceberg_type: PrimitiveType mode: MetricsMode ``` Where we can use the `field_id` to properly populate the maps, and the `iceberg_type` to feed into `to_bytes` to do the conversion (so we don't have to have yet another one for PyArrow). I did a quick test, and it seems to work: ```python def test_complex_schema(table_schema_nested: Schema): tbl = pa.Table.from_pydict({ "foo": ["a", "b"], "bar": [1, 2], "baz": [False, True], "qux": [["a", "b"], ["c", "d"]], "quux": [[("a", (("aa", 1), ("ab", 2)))], [("b", (("ba", 3), ("bb", 4)))]], "location": [[(52.377956, 4.897070), (4.897070, -122.431297)], [(43.618881, -116.215019), (41.881832, -87.623177)]], "person": [("Fokko", 33), ("Max", 42)] # Possible data quality issue }, schema=schema_to_pyarrow(table_schema_nested) ) stats_columns = pre_order_visit(table_schema_nested, PyArrowStatisticsCollector(table_schema_nested, {})) visited_paths = [] def file_visitor(written_file: Any) -> None: visited_paths.append(written_file) with TemporaryDirectory() as tmpdir: pq.write_to_dataset(tbl, tmpdir, file_visitor=file_visitor) assert visited_paths[0].metadata.num_columns == len(stats_columns) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
