RussellSpitzer commented on code in PR #9803:
URL: https://github.com/apache/iceberg/pull/9803#discussion_r1504798801
##########
spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java:
##########
@@ -1463,6 +1449,176 @@ public void testSnapshotProperty() {
assertThat(table.currentSnapshot().summary()).containsKeys(commitMetricsKeys);
}
+ @Test
+ public void testBinPackRewriterWithSpecificOutputSpec() {
+ Table table = createTable(10);
+ shouldHaveFiles(table, 10);
+ Integer previousSpecId = table.spec().specId();
+ // create multiple partition specs with different commit
+ table.updateSpec().addField(Expressions.truncate("c2", 2)).commit();
+ table.updateSpec().addField(Expressions.bucket("c3", 2)).commit();
+
+ /*
+ This is how the table.specs() look like
+ specId :: PartitionSpec
+ 0::[]
+ 1::[
+ 1000: c2_trunc_2: truncate[2](2)
+ ]
+ 2::[
+ 1000: c2_trunc_2: truncate[2](2)
+ 1001: c3_bucket_2: bucket[2](3)
+ ]
+ */
+ // rewrite to the truncate("c2", 2)
+ int outputSpecId = table.specs().size() - 2;
+
+ long dataSizeBefore = testDataSize(table);
+ long count = currentData().size();
+
+ RewriteDataFiles.Result result =
+ basicRewrite(table)
+ .option(RewriteDataFiles.OUTPUT_SPEC_ID,
String.valueOf(outputSpecId))
+ .option(SizeBasedFileRewriter.REWRITE_ALL, "true")
+ .binPack()
+ .execute();
+
+ assertThat(result.rewrittenBytesCount()).isEqualTo(dataSizeBefore);
+ assertThat(currentData().size()).isEqualTo(count);
+ shouldHaveProvidedPartitionSpec(table, outputSpecId);
+ }
+
+ @Test
+ public void testBinpackRewriteWithInvalidOutputSpecId() {
+ Table table = createTable(10);
+ shouldHaveFiles(table, 10);
+ int previousSpecId = table.spec().specId();
+ // simulate multiple partition specs with different commit
+ table.updateSpec().addField(Expressions.truncate("c2", 2)).commit();
+ table.updateSpec().addField(Expressions.bucket("c3", 2)).commit();
+ /*
+ This is how the table.specs() look like
+ specId :: ParitionSpec
+ 0::[]
+ 1::[
+ 1000: c2_trunc_2: truncate[2](2)
+ ]
+ 2::[
+ 1000: c2_trunc_2: truncate[2](2)
+ 1001: c3_bucket_2: bucket[2](3)
+ ]
+ */
+
+ Assertions.assertThatThrownBy(
+ () ->
+ actions()
+ .rewriteDataFiles(table)
+ .option(RewriteDataFiles.OUTPUT_SPEC_ID,
String.valueOf(1234))
+ .binPack()
+ .execute())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Output spec id 1234 is not a valid spec id for table");
+ }
+
+ @Test
+ public void testSortRewriterWithSpecificOutputSpecId() {
+ Table table = createTable(10);
+ shouldHaveFiles(table, 10);
+ Integer previousSpecId = table.spec().specId();
+ // simulate multiple partition specs with different commit
+ table.updateSpec().addField(Expressions.truncate("c2", 2)).commit();
+ table.updateSpec().addField(Expressions.bucket("c3", 2)).commit();
+
+ /*
+ This is how the table.specs() look like
+ specId :: ParitionSpec
+ 0::[]
+ 1::[
+ 1000: c2_trunc_2: truncate[2](2)
+ ]
+ 2::[
+ 1000: c2_trunc_2: truncate[2](2)
+ 1001: c3_bucket_2: bucket[2](3)
+ ]
+ */
+ // rewrite to the truncate("c2", 2)
+ int outputSpecId = table.specs().size() - 2;
+
+ long dataSizeBefore = testDataSize(table);
+ long count = currentData().size();
+
+ RewriteDataFiles.Result result =
+ basicRewrite(table)
+ .option(RewriteDataFiles.OUTPUT_SPEC_ID,
String.valueOf(outputSpecId))
+ .option(SizeBasedFileRewriter.REWRITE_ALL, "true")
+
.sort(SortOrder.builderFor(table.schema()).asc("c2").asc("c3").build())
+ .execute();
+
+ assertThat(result.rewrittenBytesCount()).isEqualTo(dataSizeBefore);
+ assertThat(currentData().size()).isEqualTo(count);
+ shouldHaveProvidedPartitionSpec(table, outputSpecId);
+ }
+
+ @Test
+ public void testZOrderRewriteWithSpecificOutputSpecId() {
+ Table table = createTable(10);
+ shouldHaveFiles(table, 10);
+ Integer previousSpecId = table.spec().specId();
+ // simulate multiple partition specs with different commit
+ table.updateSpec().addField(Expressions.truncate("c2", 2)).commit();
+ table.updateSpec().addField(Expressions.bucket("c3", 2)).commit();
+
+ /*
+ This is how the table.specs() look like
+ specId :: ParitionSpec
+ 0::[]
+ 1::[
+ 1000: c2_trunc_2: truncate[2](2)
+ ]
+ 2::[
+ 1000: c2_trunc_2: truncate[2](2)
+ 1001: c3_bucket_2: bucket[2](3)
+ ]
+ */
+ // rewrite to the truncate("c2", 2)
+ int outputSpecId = table.specs().size() - 2;
+
+ long dataSizeBefore = testDataSize(table);
+ long count = currentData().size();
+
+ RewriteDataFiles.Result result =
+ basicRewrite(table)
+ .option(RewriteDataFiles.OUTPUT_SPEC_ID,
String.valueOf(outputSpecId))
+ .option(SizeBasedFileRewriter.REWRITE_ALL, "true")
+ .zOrder("c2", "c3")
+ .execute();
+
+ assertThat(result.rewrittenBytesCount()).isEqualTo(dataSizeBefore);
+ assertThat(currentData().size()).isEqualTo(count);
+ shouldHaveProvidedPartitionSpec(table, outputSpecId);
+ }
+
+ protected void shouldHaveProvidedPartitionSpec(Table table, int
outputSpecId) {
Review Comment:
nit: dataFilesWrittenWithPartitionSpec(...)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
