aihuaxu commented on code in PR #14349:
URL: https://github.com/apache/iceberg/pull/14349#discussion_r2482002613
##########
spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkVariantRead.java:
##########
@@ -169,4 +161,118 @@ public void testVariantNullValueProjection(boolean
vectorized) {
assertThat(row.getLong(0)).isEqualTo(10L);
assertThat(row.isNullAt(1)).isTrue();
}
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testNestedStructVariant(boolean vectorized) {
+ assumeThat(vectorized).as("Variant vectorized Parquet read is not
implemented yet").isFalse();
+ setVectorization(vectorized);
+
+ String structTable = CATALOG + ".default.var_struct";
+ sql("DROP TABLE IF EXISTS %s", structTable);
+ sql(
+ "CREATE TABLE %s (id BIGINT, s STRUCT<v: VARIANT>) USING iceberg "
+ + "TBLPROPERTIES ('format-version'='3',
'read.parquet.vectorization.enabled'='false')",
+ structTable);
+
+ String j1 = "{\"a\":1}";
+ String j2 = "{\"b\":2}";
+ sql("INSERT INTO %s SELECT 1, named_struct('v', parse_json('%s'))",
structTable, j1);
+ sql("INSERT INTO %s SELECT 2, named_struct('v', parse_json('%s'))",
structTable, j2);
+
+ Dataset<Row> df =
+ spark.table(structTable).selectExpr("id", "to_json(s.v) as
v_json").orderBy("id");
+ java.util.List<Row> rows = df.collectAsList();
+ assertThat(rows.get(0).getLong(0)).isEqualTo(1L);
+ assertThat(rows.get(0).getString(1)).isEqualTo("{\"a\":1}");
Review Comment:
Same for the following tests.
##########
spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkVariantRead.java:
##########
@@ -169,4 +161,118 @@ public void testVariantNullValueProjection(boolean
vectorized) {
assertThat(row.getLong(0)).isEqualTo(10L);
assertThat(row.isNullAt(1)).isTrue();
}
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testNestedStructVariant(boolean vectorized) {
+ assumeThat(vectorized).as("Variant vectorized Parquet read is not
implemented yet").isFalse();
+ setVectorization(vectorized);
+
+ String structTable = CATALOG + ".default.var_struct";
+ sql("DROP TABLE IF EXISTS %s", structTable);
+ sql(
+ "CREATE TABLE %s (id BIGINT, s STRUCT<v: VARIANT>) USING iceberg "
+ + "TBLPROPERTIES ('format-version'='3',
'read.parquet.vectorization.enabled'='false')",
+ structTable);
+
+ String j1 = "{\"a\":1}";
+ String j2 = "{\"b\":2}";
+ sql("INSERT INTO %s SELECT 1, named_struct('v', parse_json('%s'))",
structTable, j1);
+ sql("INSERT INTO %s SELECT 2, named_struct('v', parse_json('%s'))",
structTable, j2);
+
+ Dataset<Row> df =
+ spark.table(structTable).selectExpr("id", "to_json(s.v) as
v_json").orderBy("id");
+ java.util.List<Row> rows = df.collectAsList();
+ assertThat(rows.get(0).getLong(0)).isEqualTo(1L);
+ assertThat(rows.get(0).getString(1)).isEqualTo("{\"a\":1}");
Review Comment:
Why do we get the string here? I think we should get variant and then
extract variant field 'a' value which is expected to be 1.
Same for line#189.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
