nssalian commented on code in PR #18851:
URL: https://github.com/apache/iceberg/pull/18851#discussion_r4579288847


##########
spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestSparkVariantRead.java:
##########
@@ -557,6 +552,66 @@ public void testReadShreddedWithMetricsDisabled(String 
metricsMode)
     sql("DROP TABLE IF EXISTS %s", noStatsTable);
   }
 
+  @ParameterizedTest
+  @ValueSource(booleans = {false, true})
+  void readShreddedArrayEnabledBySessionConf(boolean required)
+      throws IOException, NoSuchTableException, ParseException {
+    String table = CATALOG + ".default.var_shredded_array";
+    sql("DROP TABLE IF EXISTS %s", table);
+    sql(
+        "CREATE TABLE %s (id BIGINT, %s) USING iceberg TBLPROPERTIES 
('format-version'='3')",
+        table, required ? "v VARIANT NOT NULL" : "v VARIANT");
+    insertShredded(
+        "INSERT INTO %s VALUES (1, parse_json('{\"a\": [1, 2]}')), (2, 
parse_json('{\"a\": [3]}'))",
+        table);
+
+    Table icebergTable = Spark3Util.loadIcebergTable(spark, table);
+    
assertThat(icebergTable.schema().findField("v").isRequired()).isEqualTo(required);
+    assertHasTypedValueSubtree(icebergTable);
+
+    List<Row> rows = assertVectorizedMatchesRowReader(table);
+    assertThat(rows).extracting(row -> row.getLong(0)).containsExactly(1L, 2L);
+    assertLongArray(toVariant(rows.get(0)).getFieldByKey("a"), 1L, 2L);
+    assertLongArray(toVariant(rows.get(1)).getFieldByKey("a"), 3L);
+
+    sql("DROP TABLE IF EXISTS %s", table);
+  }
+
+  @Test
+  void readMixedShreddedAndUnshreddedFiles()
+      throws IOException, NoSuchTableException, ParseException {
+    String table = CATALOG + ".default.var_mixed_files";
+    sql("DROP TABLE IF EXISTS %s", table);
+    sql(
+        "CREATE TABLE %s (id BIGINT, v VARIANT) USING iceberg "
+            + "TBLPROPERTIES ('format-version'='3')",
+        table);
+    spark
+        .sql("SELECT 1L AS id, parse_json('{\"name\":\"alice\",\"age\":30}') 
AS v")
+        .coalesce(1)
+        .writeTo(table)
+        .append();
+    insertShredded(
+        "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM ("
+            + "SELECT 2L AS id, CAST(NULL AS VARIANT) AS v "
+            + "UNION ALL SELECT 3L, parse_json('{\"a\":[1,2]}') "
+            + "UNION ALL SELECT 4L, parse_json('{\"a\":[3]}'))",
+        table);
+
+    Table icebergTable = Spark3Util.loadIcebergTable(spark, table);
+    assertThat(shreddedFiles(icebergTable)).containsExactlyInAnyOrder(false, 
true);
+    sql("ALTER TABLE %s SET TBLPROPERTIES 
('read.parquet.vectorization.batch-size'='2')", table);

Review Comment:
   Makes sense. Added readShreddedVariantAcrossRowGroups in 4.0/4.1/4.2. It 
forces one record per row group (write.parquet.row-group-size-bytes=1 with 
check counts 1), asserts the shredded file has more than one row group, and 
compares vectorized and row-reader  results. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to