RussellSpitzer commented on code in PR #18477:
URL: https://github.com/apache/iceberg/pull/18477#discussion_r4518764508


##########
spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java:
##########
@@ -1416,4 +1420,125 @@ public void testCleanExpiredMetadata() {
         .contains(FILE_A.location())
         .doesNotContain(fileInNewSpec.location());
   }
+
+  @TestTemplate
+  public void 
testExpireWithMissingManifestListReachableOnlyFromExpiredSnapshots() {
+    assertExpireSkipsMissingMetadata(Snapshot::manifestListLocation);
+  }
+
+  @TestTemplate
+  public void testExpireWithMissingManifestReachableOnlyFromExpiredSnapshots() 
{
+    assertExpireSkipsMissingMetadata(
+        snapshot -> 
Iterables.getOnlyElement(snapshot.allManifests(table.io())).path());
+  }
+
+  private void assertExpireSkipsMissingMetadata(
+      Function<Snapshot, String> missingLocationResolver) {
+    table.newAppend().appendFile(FILE_A).commit();
+    String missingLocation = 
missingLocationResolver.apply(table.currentSnapshot());
+
+    table.newOverwrite().deleteFile(FILE_A).addFile(FILE_B).commit();
+    table.newOverwrite().deleteFile(FILE_B).addFile(FILE_C).commit();
+    table.newAppend().appendFile(FILE_D).commit();
+
+    long end = rightAfterSnapshot();
+
+    table.io().deleteFile(missingLocation);
+
+    Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
+
+    ExpireSnapshots.Result result =
+        SparkActions.get()
+            .expireSnapshots(table)
+            .expireOlderThan(end)
+            .deleteWith(deletedFiles::add)
+            .execute();
+
+    assertThat(table.snapshots()).as("Expiration should still 
commit").hasSize(1);
+    assertThat(result.deletedDataFilesCount())
+        .as("Should report the files deleted despite the missing metadata")
+        .isEqualTo(1L);
+    assertThat(deletedFiles)
+        .as("Should delete files resolved from other expired snapshots")
+        .contains(FILE_B.location());
+    assertThat(deletedFiles)
+        .as("Should leave files reachable only through the missing metadata as 
orphans")
+        .doesNotContain(FILE_A.location());
+  }
+
+  @TestTemplate
+  public void 
testExpireDeletesStatisticsFileOfSnapshotWithMissingManifestList() {
+    table.newAppend().appendFile(FILE_A).commit();
+    Snapshot firstSnapshot = table.currentSnapshot();
+    String statisticsLocation = tableLocation + "/metadata/stats-" + 
UUID.randomUUID();
+    table
+        .updateStatistics()
+        .setStatistics(
+            new GenericStatisticsFile(
+                firstSnapshot.snapshotId(), statisticsLocation, 1L, 1L, 
List.of()))
+        .commit();
+
+    table.newAppend().appendFile(FILE_B).commit();
+
+    long end = rightAfterSnapshot();
+
+    table.io().deleteFile(firstSnapshot.manifestListLocation());
+
+    Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
+
+    SparkActions.get()
+        .expireSnapshots(table)
+        .expireOlderThan(end)
+        .deleteWith(deletedFiles::add)
+        .execute();
+
+    assertThat(deletedFiles)
+        .as("Should delete the statistics file even though the manifest list 
is missing")
+        .contains(statisticsLocation);
+  }
+
+  @TestTemplate
+  public void 
testExpireFailsFastWhenMissingManifestReachableFromRetainedSnapshot() {

Review Comment:
   not sure "fails fast" is informative. I'd probably drop that



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to