kevinjqliu commented on code in PR #1498:
URL: https://github.com/apache/iceberg-python/pull/1498#discussion_r1912521440
##########
tests/io/test_pyarrow_visitor.py:
##########
@@ -625,6 +626,91 @@ def
test_pyarrow_schema_ensure_large_types(pyarrow_schema_nested_without_ids: pa
assert
_pyarrow_schema_ensure_large_types(pyarrow_schema_nested_without_ids) ==
expected_schema
+def test_pyarrow_schema_unsupported_type() -> None:
+ unsupported_field = pa.field("latitude", pa.decimal256(20, 26),
nullable=False, metadata={"PARQUET:field_id": "2"})
+ schema = pa.schema(
+ [
+ pa.field("foo", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "1"}),
+ pa.field(
+ "location",
+ pa.large_list(
+ pa.field(
+ "item",
+ pa.struct(
+ [
+ unsupported_field,
+ pa.field("longitude", pa.float32(),
nullable=False, metadata={"PARQUET:field_id": "3"}),
+ ]
+ ),
+ metadata={"PARQUET:field_id": "4"},
+ )
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "5"},
+ ),
+ ],
+ metadata={"PARQUET:field_id": "6"},
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException, match=re.escape("Column 'latitude'
has an unsupported type: decimal256(20, 26)")
+ ) as exc_info:
+ pyarrow_to_schema(schema)
+ assert exc_info.value.field == unsupported_field
+ exception_cause = exc_info.value.__cause__
+ assert isinstance(exception_cause, TypeError)
+ assert "Unsupported type: decimal256(20, 26)" in exception_cause.args[0]
+
+ unsupported_field = pa.field(
+ "quux",
+ pa.map_(
+ pa.field("key", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "2"}),
+ pa.field(
+ "value",
+ pa.map_(
+ pa.field("key", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "5"}),
+ pa.field("value", pa.decimal256(2, 3),
metadata={"PARQUET:field_id": "6"}),
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "4"},
+ ),
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "3"},
+ )
+ schema = pa.schema(
+ [
+ pa.field("foo", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "1"}),
+ unsupported_field,
+ ]
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException,
+ match=re.escape("Column 'quux' has an unsupported type: map<string,
map<string, decimal256(2, 3)>>"),
Review Comment:
Thanks for the explanation!
The same issue is in the above example with the `latitude` field. `latitude`
is nested deep in the schema but the error message only mention that the field
is unsupported.
I think ideally we might want to return the path to the field. For example,
`foo.location.item.latitude`
But for now i think this is fine. It's better than the existing behavior
##########
tests/integration/test_add_files.py:
##########
@@ -616,13 +616,18 @@ def
test_add_files_with_timestamp_tz_ns_fails(session_catalog: Catalog, format_v
# add the parquet files as data files
with pytest.raises(
- TypeError,
- match=re.escape(
- "Iceberg does not yet support 'ns' timestamp precision. Use
'downcast-ns-timestamp-to-us-on-write' configuration property to automatically
downcast 'ns' to 'us' on write."
- ),
- ):
+ UnsupportedPyArrowTypeException,
+ match=re.escape("Column 'quux' has an unsupported type: timestamp[ns,
tz=UTC]"),
+ ) as exc_info:
tbl.add_files(file_paths=[file_path])
+ exception_cause = exc_info.value.__cause__
+ assert isinstance(exception_cause, TypeError)
+ assert (
+ "Iceberg does not yet support 'ns' timestamp precision. Use
'downcast-ns-timestamp-to-us-on-write' configuration property to automatically
downcast 'ns' to 'us' on write."
Review Comment:
is this part of the error message? if so, can we add it to the
`pytest.raises` to be explicit
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
