kevinjqliu commented on code in PR #1264:
URL: https://github.com/apache/iceberg-python/pull/1264#discussion_r1821237947
##########
tests/integration/test_writes/test_writes.py:
##########
@@ -1448,3 +1448,44 @@ def
test_rewrite_manifest_after_partition_evolution(session_catalog: Catalog) ->
EqualTo("category", "A"),
),
)
+
+
+@pytest.mark.integration
+def test_writing_null_structs(session_catalog: Catalog) -> None:
+ import pyarrow as pa
+
+ schema = pa.schema([
+ pa.field(
+ "struct_field_1",
+ pa.struct([
+ pa.field("string_nested_1", pa.string()),
+ pa.field("int_item_2", pa.int32()),
+ pa.field("float_item_2", pa.float32()),
+ ]),
+ ),
+ ])
+
+ records = [
+ {
+ "struct_field_1": {
+ "string_nested_1": "nest_1",
+ "int_item_2": 1234,
+ "float_item_2": 1.234,
+ },
+ },
+ {},
+ ]
+
+ try:
+ session_catalog.drop_table(
+ identifier="default.test_writing_null_structs",
+ )
+ except NoSuchTableError:
+ pass
+
+ table = session_catalog.create_table("default.test_writing_null_structs",
schema)
+
+ pyarrow_table: pa.Table = pa.Table.from_pylist(records, schema=schema)
+ table.append(pyarrow_table)
+
+ assert pyarrow_table.to_pandas()["struct_field_1"].tolist() ==
table.scan().to_pandas()["struct_field_1"].tolist()
Review Comment:
Looks like the null masking is also done by pandas.
```
>>> pyarrow_table['struct_field_1']
<pyarrow.lib.ChunkedArray object at 0x10482c6d0>
[
-- is_valid:
[
false
]
-- child 0 type: string
[
""
]
-- child 1 type: int32
[
0
]
-- child 2 type: float
[
0
]
]
```
```
>>> pyarrow_table['struct_field_1'].to_pandas()
0 None
Name: struct_field_1, dtype: object
```
##########
pyiceberg/io/pyarrow.py:
##########
@@ -1807,7 +1807,11 @@ def struct(
else:
raise ResolveError(f"Field is required, and could not be found
in the file: {field}")
- return pa.StructArray.from_arrays(arrays=field_arrays,
fields=pa.struct(fields))
+ return pa.StructArray.from_arrays(
+ arrays=field_arrays,
+ fields=pa.struct(fields),
+ mask=struct_array.is_null() if isinstance(struct_array,
pa.StructArray) else None,
Review Comment:
Very interesting, i didnt know about this
https://arrow.apache.org/docs/python/generated/pyarrow.StructArray.html#pyarrow.StructArray.from_arrays
https://arrow.apache.org/docs/python/generated/pyarrow.StructArray.html#pyarrow.StructArray.is_null
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
