huaxingao commented on code in PR #18851:
URL: https://github.com/apache/iceberg/pull/18851#discussion_r4577143210
##########
spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkVariantRead.java:
##########
@@ -557,6 +552,66 @@ public void testReadShreddedWithMetricsDisabled(String
metricsMode)
sql("DROP TABLE IF EXISTS %s", noStatsTable);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void readShreddedArrayEnabledBySessionConf(boolean required)
+ throws IOException, NoSuchTableException, ParseException {
+ String table = CATALOG + ".default.var_shredded_array";
+ sql("DROP TABLE IF EXISTS %s", table);
+ sql(
+ "CREATE TABLE %s (id BIGINT, %s) USING iceberg TBLPROPERTIES
('format-version'='3')",
+ table, required ? "v VARIANT NOT NULL" : "v VARIANT");
+ insertShredded(
+ "INSERT INTO %s VALUES (1, parse_json('{\"a\": [1, 2]}')), (2,
parse_json('{\"a\": [3]}'))",
+ table);
+
+ Table icebergTable = Spark3Util.loadIcebergTable(spark, table);
+
assertThat(icebergTable.schema().findField("v").isRequired()).isEqualTo(required);
+ assertHasTypedValueSubtree(icebergTable);
+
+ List<Row> rows = assertVectorizedMatchesRowReader(table);
+ assertThat(rows).extracting(row -> row.getLong(0)).containsExactly(1L, 2L);
+ assertLongArray(toVariant(rows.get(0)).getFieldByKey("a"), 1L, 2L);
+ assertLongArray(toVariant(rows.get(1)).getFieldByKey("a"), 3L);
+
+ sql("DROP TABLE IF EXISTS %s", table);
+ }
+
+ @Test
+ void readMixedShreddedAndUnshreddedFiles()
+ throws IOException, NoSuchTableException, ParseException {
+ String table = CATALOG + ".default.var_mixed_files";
+ sql("DROP TABLE IF EXISTS %s", table);
+ sql(
+ "CREATE TABLE %s (id BIGINT, v VARIANT) USING iceberg "
+ + "TBLPROPERTIES ('format-version'='3')",
+ table);
+ spark
+ .sql("SELECT 1L AS id, parse_json('{\"name\":\"alice\",\"age\":30}')
AS v")
+ .coalesce(1)
+ .writeTo(table)
+ .append();
+ insertShredded(
+ "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM ("
+ + "SELECT 2L AS id, CAST(NULL AS VARIANT) AS v "
+ + "UNION ALL SELECT 3L, parse_json('{\"a\":[1,2]}') "
+ + "UNION ALL SELECT 4L, parse_json('{\"a\":[3]}'))",
+ table);
+
+ Table icebergTable = Spark3Util.loadIcebergTable(spark, table);
+ assertThat(shreddedFiles(icebergTable)).containsExactlyInAnyOrder(false,
true);
+ sql("ALTER TABLE %s SET TBLPROPERTIES
('read.parquet.vectorization.batch-size'='2')", table);
Review Comment:
This splits the file into batches, but each file still has one row group, so
the row-based reader never switches row groups (setRowGroupInfo ->
setPageSource). Can we add a test with more than one row group?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]