This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new f5704ee94b Core: v4: Prune leaf manifests with no live files in
FilePlanner (#18725)
f5704ee94b is described below
commit f5704ee94b62839d79d9caf38045a3275fd3b4f0
Author: Anoop Johnson <[email protected]>
AuthorDate: Mon Nov 2 12:41:33 2026 -0800
Core: v4: Prune leaf manifests with no live files in FilePlanner (#18725)
Scan planning returns only live files (ADDED, EXISTING, or MODIFIED).
Skip expanding a leaf manifest whose manifest_info reports none of these
live file counts: the leaf is never opened and is counted as a skipped
data manifest.
---
.../main/java/org/apache/iceberg/FilePlanner.java | 14 +++-
.../java/org/apache/iceberg/TestFilePlanner.java | 78 ++++++++++++++++++----
2 files changed, 78 insertions(+), 14 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/FilePlanner.java
b/core/src/main/java/org/apache/iceberg/FilePlanner.java
index ca63c2b64f..12d93c8cd2 100644
--- a/core/src/main/java/org/apache/iceberg/FilePlanner.java
+++ b/core/src/main/java/org/apache/iceberg/FilePlanner.java
@@ -140,11 +140,15 @@ class FilePlanner {
List<CloseableIterable<TrackedFile>> leafPlanTasks = Lists.newArrayList();
for (ManifestFile leaf : leafManifests) {
- leafPlanTasks.add(reader(leaf));
+ if (hasLiveFiles(leaf)) {
+ leafPlanTasks.add(reader(leaf));
+ } else {
+ scanMetrics.skippedDataManifests().increment();
+ }
}
CloseableIterable<TrackedFile> leafFiles =
- executorService != null && leafManifests.size() > 1
+ executorService != null && leafPlanTasks.size() > 1
? new ParallelIterable<>(leafPlanTasks, executorService)
: CloseableIterable.concat(leafPlanTasks);
@@ -204,6 +208,12 @@ class FilePlanner {
ResidualEvaluator.of(spec, filter, caseSensitive));
}
+ private static boolean hasLiveFiles(ManifestFile leaf) {
+ return leaf.hasAddedFiles()
+ || leaf.hasExistingFiles()
+ || (leaf.modifiedFilesCount() != null && leaf.modifiedFilesCount() >
0);
+ }
+
/** Per-spec task inputs computed once and shared across all files of a
spec. */
private static class TaskContext {
private final String schemaAsString;
diff --git a/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
b/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
index 318fe7df01..9a52fffd66 100644
--- a/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
+++ b/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
@@ -517,6 +517,47 @@ class TestFilePlanner {
}
}
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ void leafReportingNoLiveFilesIsPruned(FileFormat format) throws IOException {
+ ManifestFile addedLeaf =
+ writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(unpartitionedDataFile(FILE_A)));
+ ManifestFile existingLeaf =
+ writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(unpartitionedDataFile(FILE_B)));
+ ManifestFile modifiedLeaf =
+ writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(unpartitionedDataFile(FILE_C)));
+ String deadLeafLocation =
+ "s3://bucket/db/table/dead-leaf." +
format.name().toLowerCase(Locale.ROOT);
+ ManifestFile root =
+ writeManifest(
+ format,
+ EMPTY_PARTITION,
+ ImmutableList.of(
+ manifestRef(FileContent.DATA_MANIFEST, addedLeaf.path(),
addedOnly(1)),
+ manifestRef(FileContent.DATA_MANIFEST, existingLeaf.path(),
existingOnly(1)),
+ manifestRef(FileContent.DATA_MANIFEST, modifiedLeaf.path(),
modifiedOnly(1)),
+ manifestRef(FileContent.DATA_MANIFEST, deadLeafLocation,
deletedOnly(2))));
+
+ FileIO io = Mockito.spy(fileIO);
+ ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext());
+ FilePlanner planner =
+ new FilePlanner(io, root, TABLE_SCHEMA,
UNPARTITIONED_SPECS).scanMetrics(metrics);
+
+ List<FileScanTask> tasks;
+ try (CloseableIterable<FileScanTask> plan = planner.planFiles()) {
+ tasks = Lists.newArrayList(plan);
+ }
+
+ assertThat(tasks)
+ .extracting(task -> task.file().location())
+ .as("leaves with added, existing, or modified files are expanded;
deleted-only is pruned")
+ .containsExactlyInAnyOrder(FILE_A, FILE_B, FILE_C);
+ Mockito.verify(io, Mockito.never()).newInputFile(deadLeafLocation);
+ assertThat(metrics.skippedDataManifests().value())
+ .as("the pruned leaf is counted as a skipped data manifest")
+ .isEqualTo(1L);
+ }
+
@ParameterizedTest
@FieldSource("MANIFEST_FORMATS")
void deleteManifestInRootUnsupported(FileFormat format) throws IOException {
@@ -537,7 +578,7 @@ class TestFilePlanner {
void v3ManifestLeafUnsupported(FileFormat format) throws IOException {
String leafLocation = "s3://bucket/db/table/v3-leaf." +
format.name().toLowerCase(Locale.ROOT);
TrackedFile v3Leaf =
- manifestRef(FileContent.DATA_MANIFEST, leafLocation,
dataManifestInfo(1, 0));
+ manifestRef(FileContent.DATA_MANIFEST, leafLocation, manifestInfo(1,
0, 0, 0, 0));
ManifestFile root = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(v3Leaf));
FilePlanner planner = new FilePlanner(fileIO, root, TABLE_SCHEMA,
UNPARTITIONED_SPECS);
@@ -636,11 +677,11 @@ class TestFilePlanner {
}
private static TrackedFile dataManifest(String location, int addedFiles) {
- return manifestRef(FileContent.DATA_MANIFEST, location,
dataManifestInfo(addedFiles));
+ return manifestRef(FileContent.DATA_MANIFEST, location,
addedOnly(addedFiles));
}
private static TrackedFile dataManifestWithStats(String location,
ContentStats stats) {
- return manifestRefWithStats(FileContent.DATA_MANIFEST, location, stats,
dataManifestInfo(1));
+ return manifestRefWithStats(FileContent.DATA_MANIFEST, location, stats,
addedOnly(1));
}
private static Integer lowerBound(DataFile file, int fieldId) {
@@ -676,22 +717,35 @@ class TestFilePlanner {
0L); // totalBytes
}
- private static ManifestInfo dataManifestInfo(int addedFiles) {
- return dataManifestInfo(addedFiles, FORMAT_VERSION_V4);
+ private static ManifestInfo addedOnly(int addedFiles) {
+ return manifestInfo(addedFiles, 0, 0, 0, FORMAT_VERSION_V4);
+ }
+
+ private static ManifestInfo existingOnly(int existingFiles) {
+ return manifestInfo(0, existingFiles, 0, 0, FORMAT_VERSION_V4);
+ }
+
+ private static ManifestInfo modifiedOnly(int modifiedFiles) {
+ return manifestInfo(0, 0, modifiedFiles, 0, FORMAT_VERSION_V4);
+ }
+
+ private static ManifestInfo deletedOnly(int deletedFiles) {
+ return manifestInfo(0, 0, 0, deletedFiles, FORMAT_VERSION_V4);
}
- private static ManifestInfo dataManifestInfo(int addedFiles, int
formatVersion) {
+ private static ManifestInfo manifestInfo(
+ int addedFiles, int existingFiles, int modifiedFiles, int deletedFiles,
int formatVersion) {
return ManifestInfoStruct.builder()
.addedFilesCount(addedFiles)
- .existingFilesCount(0)
- .deletedFilesCount(0)
+ .existingFilesCount(existingFiles)
+ .deletedFilesCount(deletedFiles)
.replacedFilesCount(0)
- .modifiedFilesCount(0)
+ .modifiedFilesCount(modifiedFiles)
.addedRowsCount(addedFiles * RECORD_COUNT)
- .existingRowsCount(0)
- .deletedRowsCount(0)
+ .existingRowsCount(existingFiles * RECORD_COUNT)
+ .deletedRowsCount(deletedFiles * RECORD_COUNT)
.replacedRowsCount(0)
- .modifiedRowsCount(0)
+ .modifiedRowsCount(modifiedFiles * RECORD_COUNT)
.minSequenceNumber(0L)
.formatVersion(formatVersion)
.build();
