kevinjqliu commented on code in PR #1498:
URL: https://github.com/apache/iceberg-python/pull/1498#discussion_r1912158909
##########
tests/io/test_pyarrow_visitor.py:
##########
@@ -625,6 +626,91 @@ def
test_pyarrow_schema_ensure_large_types(pyarrow_schema_nested_without_ids: pa
assert
_pyarrow_schema_ensure_large_types(pyarrow_schema_nested_without_ids) ==
expected_schema
+def test_pyarrow_schema_unsupported_type() -> None:
+ unsupported_field = pa.field("latitude", pa.decimal256(20, 26),
nullable=False, metadata={"PARQUET:field_id": "2"})
+ schema = pa.schema(
+ [
+ pa.field("foo", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "1"}),
+ pa.field(
+ "location",
+ pa.large_list(
+ pa.field(
+ "item",
+ pa.struct(
+ [
+ unsupported_field,
+ pa.field("longitude", pa.float32(),
nullable=False, metadata={"PARQUET:field_id": "3"}),
+ ]
+ ),
+ metadata={"PARQUET:field_id": "4"},
+ )
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "5"},
+ ),
+ ],
+ metadata={"PARQUET:field_id": "6"},
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException, match=re.escape("Column 'latitude'
has an unsupported type: decimal256(20, 26)")
+ ) as exc_info:
+ pyarrow_to_schema(schema)
+ assert exc_info.value.field == unsupported_field
+ exception_cause = exc_info.value.__cause__
+ assert isinstance(exception_cause, TypeError)
+ assert "Unsupported type: decimal256(20, 26)" in exception_cause.args[0]
+
+ unsupported_field = pa.field(
+ "quux",
+ pa.map_(
+ pa.field("key", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "2"}),
+ pa.field(
+ "value",
+ pa.map_(
+ pa.field("key", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "5"}),
+ pa.field("value", pa.decimal256(2, 3),
metadata={"PARQUET:field_id": "6"}),
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "4"},
+ ),
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "3"},
+ )
+ schema = pa.schema(
+ [
+ pa.field("foo", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "1"}),
+ unsupported_field,
+ ]
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException,
+ match=re.escape("Column 'quux' has an unsupported type: map<string,
map<string, decimal256(2, 3)>>"),
+ ) as exc_info:
+ pyarrow_to_schema(schema)
+ assert exc_info.value.field == unsupported_field
+ exception_cause = exc_info.value.__cause__
+ assert isinstance(exception_cause, TypeError)
+ assert "Unsupported type: decimal256(2, 3)" in exception_cause.args[0]
+
+ unsupported_field = pa.field("foo", pa.timestamp(unit="ns"),
nullable=False, metadata={"PARQUET:field_id": "1"})
+ schema = pa.schema(
+ [
+ unsupported_field,
+ pa.field("bar", pa.int32(), nullable=False,
metadata={"PARQUET:field_id": "2"}),
+ ]
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException,
+ match=re.escape("Column 'foo' has an unsupported type: timestamp[ns]"),
+ ) as exc_info:
Review Comment:
this is only true when `_downcast_ns_timestamp_to_us` is set not set or set
to False
https://github.com/apache/iceberg-python/blob/cad0ad7d9358315abe1315de2a64227d91acceaa/pyiceberg/io/pyarrow.py#L1174-L1179
##########
tests/io/test_pyarrow_visitor.py:
##########
@@ -625,6 +626,91 @@ def
test_pyarrow_schema_ensure_large_types(pyarrow_schema_nested_without_ids: pa
assert
_pyarrow_schema_ensure_large_types(pyarrow_schema_nested_without_ids) ==
expected_schema
+def test_pyarrow_schema_unsupported_type() -> None:
+ unsupported_field = pa.field("latitude", pa.decimal256(20, 26),
nullable=False, metadata={"PARQUET:field_id": "2"})
+ schema = pa.schema(
+ [
+ pa.field("foo", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "1"}),
+ pa.field(
+ "location",
+ pa.large_list(
+ pa.field(
+ "item",
+ pa.struct(
+ [
+ unsupported_field,
+ pa.field("longitude", pa.float32(),
nullable=False, metadata={"PARQUET:field_id": "3"}),
+ ]
+ ),
+ metadata={"PARQUET:field_id": "4"},
+ )
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "5"},
+ ),
+ ],
+ metadata={"PARQUET:field_id": "6"},
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException, match=re.escape("Column 'latitude'
has an unsupported type: decimal256(20, 26)")
+ ) as exc_info:
+ pyarrow_to_schema(schema)
+ assert exc_info.value.field == unsupported_field
+ exception_cause = exc_info.value.__cause__
+ assert isinstance(exception_cause, TypeError)
+ assert "Unsupported type: decimal256(20, 26)" in exception_cause.args[0]
+
+ unsupported_field = pa.field(
+ "quux",
+ pa.map_(
+ pa.field("key", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "2"}),
+ pa.field(
+ "value",
+ pa.map_(
+ pa.field("key", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "5"}),
+ pa.field("value", pa.decimal256(2, 3),
metadata={"PARQUET:field_id": "6"}),
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "4"},
+ ),
+ ),
+ nullable=False,
+ metadata={"PARQUET:field_id": "3"},
+ )
+ schema = pa.schema(
+ [
+ pa.field("foo", pa.string(), nullable=False,
metadata={"PARQUET:field_id": "1"}),
+ unsupported_field,
+ ]
+ )
+ with pytest.raises(
+ UnsupportedPyArrowTypeException,
+ match=re.escape("Column 'quux' has an unsupported type: map<string,
map<string, decimal256(2, 3)>>"),
Review Comment:
why does this test return the whole map instead of just `pa.field("value",
pa.decimal256(2, 3),` like the example above?
##########
pyiceberg/io/pyarrow.py:
##########
@@ -1003,6 +1000,20 @@ def _(obj: pa.DictionaryType, visitor:
PyArrowSchemaVisitor[T]) -> T:
return visit_pyarrow(obj.value_type, visitor)
+@visit_pyarrow.register(pa.Field)
Review Comment:
that makes sense, lets keep it as is
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
