szehon-ho commented on code in PR #7651:
URL: https://github.com/apache/iceberg/pull/7651#discussion_r1199508255
##########
spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java:
##########
@@ -739,4 +789,43 @@ private void checkResult(
size(newDeletes),
result.rewriteResults().stream().mapToLong(FileGroupRewriteResult::addedBytesCount).sum());
}
+
+ private void checkSequenceNumbers(
+ Table table, List<DeleteFile> rewrittenDeletes, List<DeleteFile>
addedDeletes) {
+ StructLikeMap<List<DeleteFile>> rewrittenFilesPerPartition =
+ groupPerPartition(table, rewrittenDeletes);
+ StructLikeMap<List<DeleteFile>> addedFilesPerPartition =
groupPerPartition(table, addedDeletes);
+ for (StructLike partition : rewrittenFilesPerPartition.keySet()) {
+ Long maxRewrittenSeq =
+ rewrittenFilesPerPartition.get(partition).stream()
+ .map(ContentFile::dataSequenceNumber)
+ .max(Long::compare)
+ .get();
+ List<DeleteFile> addedPartitionFiles =
addedFilesPerPartition.get(partition);
+ if (addedPartitionFiles != null) {
+ addedPartitionFiles.forEach(
+ d ->
+ Assert.assertEquals(
+ "Sequence number should be max of rewritten set",
+ d.dataSequenceNumber(),
+ maxRewrittenSeq));
+ }
+ }
+ }
+
+ private StructLikeMap<List<DeleteFile>> groupPerPartition(
+ Table table, List<DeleteFile> deleteFiles) {
+ StructLikeMap<List<DeleteFile>> result =
+ StructLikeMap.create(Partitioning.partitionType(table));
+ for (DeleteFile deleteFile : deleteFiles) {
+ StructLike partition = deleteFile.partition();
+ List<DeleteFile> partitionFiles = result.get(partition);
+ if (partitionFiles == null) {
+ partitionFiles = Lists.newArrayList();
+ }
Review Comment:
Normally agree, but this is one of those create if not exist for the map
key, so logically looked like it fit together here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
