slfan1989 commented on code in PR #13956:
URL: https://github.com/apache/iceberg/pull/13956#discussion_r2322399508
##########
spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java:
##########
@@ -902,36 +900,59 @@ public void testInvalidArgs() {
}
@Test
- public void testPartitionStatisticFile() throws IOException {
+ public void testTableWithManyPartitionStatisticFile() throws IOException {
String sourceTableLocation = newTableLocation();
Map<String, String> properties = Maps.newHashMap();
properties.put("format-version", "2");
String tableName = "v2tblwithPartStats";
Table sourceTable =
- createMetastoreTable(sourceTableLocation, properties, "default",
tableName, 0);
+ createMetastoreTable(sourceTableLocation, properties, "default",
tableName, 0, "c1");
- TableMetadata metadata = currentMetadata(sourceTable);
- TableMetadata withPartStatistics =
- TableMetadata.buildFrom(metadata)
- .setPartitionStatistics(
- ImmutableGenericPartitionStatisticsFile.builder()
- .snapshotId(11L)
- .path("/some/partition/stats/file.parquet")
- .fileSizeInBytes(42L)
- .build())
- .build();
+ int iterations = 10;
+ for (int i = 0; i < iterations; i++) {
+ sql("insert into hive.default.%s values (%s, 'AAAAAAAAAA', 'AAAA')",
tableName, i);
+ sourceTable.refresh();
+ actions().computePartitionStats(sourceTable).execute();
+ }
- OutputFile file =
sourceTable.io().newOutputFile(metadata.metadataFileLocation());
- TableMetadataParser.overwrite(withPartStatistics, file);
+ sourceTable.refresh();
+ assertThat(sourceTable.partitionStatisticsFiles()).hasSize(iterations);
- assertThatThrownBy(
- () ->
- actions()
- .rewriteTablePath(sourceTable)
- .rewriteLocationPrefix(sourceTableLocation,
targetTableLocation())
- .execute())
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Partition statistics files are not supported
yet");
+ String targetTableLocation = targetTableLocation();
+ RewriteTablePath.Result result =
+ actions()
+ .rewriteTablePath(sourceTable)
+ .rewriteLocationPrefix(sourceTableLocation, targetTableLocation)
+ .execute();
+ checkFileNum(
+ iterations * 2 + 1, iterations, iterations, 0, iterations, iterations
* 6 + 1, result);
+
+ // Read the file list to verify statistics file paths
+ List<Tuple2<String, String>> filesToMove =
readPathPairList(result.fileListLocation());
Review Comment:
Thank you for reviewing the code! I have optimized it according to your
suggestions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
