Aulust opened a new issue, #9091: URL: https://github.com/apache/iceberg/issues/9091
### Feature Request / Improvement The problem is the same as has been discussed in this issue https://github.com/apache/iceberg/issues/510 and addressed in this pull request https://github.com/apache/iceberg/pull/514 : Spark, when reading schema from avro/parquet/etc files, makes all fields optional (nullable). Additionally, arrays and maps are allowed to have null elements. `check-nullability` addresses the case of fields, but not arrays/maps, making writing to iceberg table fail with `elements should be required, but are optional` Implementation of this change would be something like: ``` diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 502e52c34..02ab394ec 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -211,7 +211,7 @@ public class CheckCompatibility extends TypeUtil.CustomOrderSchemaVisitor<List<S this.currentType = list.elementType(); try { - if (readList.isElementRequired() && list.isElementOptional()) { + if (checkNullability && readList.isElementRequired() && list.isElementOptional()) { errors.add(": elements should be required, but are optional"); } @@ -234,7 +234,7 @@ public class CheckCompatibility extends TypeUtil.CustomOrderSchemaVisitor<List<S List<String> errors = Lists.newArrayList(); try { - if (readMap.isValueRequired() && map.isValueOptional()) { + if (checkNullability && readMap.isValueRequired() && map.isValueOptional()) { errors.add(": values should be required, but are optional"); } diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 7f5948bd5..2a1bdf625 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -525,4 +525,36 @@ public class TestReadabilityChecks { List<String> errors = CheckCompatibility.typeCompatibilityErrors(read, write); assertThat(errors).isEmpty(); } + + @Test + public void testCheckNullabilityRequiredListElements() { + Schema write = + new Schema( + required(0, "list_field", Types.ListType.ofOptional(1, Types.IntegerType.get()))); + Schema read = + new Schema( + required(0, "list_field", Types.ListType.ofRequired(1, Types.IntegerType.get()))); + + List<String> errors = CheckCompatibility.typeCompatibilityErrors(read, write); + assertThat(errors).isEmpty(); + } + + @Test + public void testCheckNullabilityRequiredMapElements() { + Schema write = + new Schema( + required( + 0, + "map_field", + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.IntegerType.get()))); + Schema read = + new Schema( + required( + 0, + "map_field", + Types.MapType.ofRequired(1, 2, Types.StringType.get(), Types.IntegerType.get()))); + + List<String> errors = CheckCompatibility.typeCompatibilityErrors(read, write); + assertThat(errors).isEmpty(); + } } ``` ### Query engine None -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
