This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e39d8178d1 [core] Fix manifest sort when max-rewrite-size is smaller
than manifest.target-file-size (#8422)
e39d8178d1 is described below
commit e39d8178d18e583be95533fc2c9b683e3a13c31d
Author: umi <[email protected]>
AuthorDate: Thu Jul 2 14:26:42 2026 +0800
[core] Fix manifest sort when max-rewrite-size is smaller than
manifest.target-file-size (#8422)
when max-rewrite-size is smaller than manifest.target-file-size, force
the addition of at least two manifest files.
---
.../paimon/operation/ManifestFileSorter.java | 9 +++-
.../paimon/manifest/ManifestFileMetaTest.java | 50 ++++++++++++++++++++++
2 files changed, 58 insertions(+), 1 deletion(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
index 6201029968..9bf96538e9 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
@@ -805,6 +805,10 @@ public class ManifestFileSorter {
* Split a section at the rewrite budget boundary: sort and rewrite the
head part that fits
* within the remaining budget, and return the remaining tail as a new
Section (or null if the
* whole section fits and no tail is left).
+ *
+ * <p>The budget is applied at manifest-file granularity. The first file
that exceeds the
+ * remaining budget is still included, and at least two files are
rewritten. Otherwise a very
+ * small {@code manifest-sort.max-rewrite-size} would make no progress.
*/
private static Section splitSectionAndRewriteHead(
Section section,
@@ -824,7 +828,10 @@ public class ManifestFileSorter {
boolean tailHasUnsortedCompactMeta = false;
for (ManifestFileMeta file : section.files) {
- if (headSize + file.fileSize() <= remainingBudget) {
+ // Rewrite budget is enforced at manifest-file granularity.
Include the first file that
+ // crosses the byte budget, and keep at least two files in the
rewrite head; otherwise a
+ // too-small budget may produce an empty or single-file head and
make no sort progress.
+ if (headSize <= remainingBudget || headFiles.size() < 2) {
headFiles.add(file);
headSize += file.fileSize();
} else {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
index eced1bf4ef..4c719dfb16 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
@@ -961,6 +961,56 @@ public class ManifestFileMetaTest extends
ManifestFileMetaTestBase {
}
}
+ @Test
+ public void
testManifestSortMaxRewriteSizeSmallerThanTargetFileSizeStillRewrites() {
+ List<ManifestFileMeta> input = new ArrayList<>();
+ for (int manifest = 0; manifest < 5; manifest++) {
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int partition = manifest; partition <= manifest + 20;
partition++) {
+ entries.add(
+ makeEntry(true, String.format("m%d-p%d", manifest,
partition), partition));
+ }
+ input.add(makeManifest(entries.toArray(new ManifestEntry[0])));
+ }
+
+ Set<String> inputManifestFileNames =
+
input.stream().map(ManifestFileMeta::fileName).collect(Collectors.toSet());
+
+ Options testOptions = new Options();
+ testOptions.set("manifest-sort.enabled", "true");
+ testOptions.set("manifest.target-file-size", "2B");
+ testOptions.set("manifest-sort.max-rewrite-size", "1B");
+
+ List<ManifestFileMeta> merged =
+ ManifestFileMerger.merge(
+ input,
+ manifestFile,
+ getPartitionType(),
+ CoreOptions.fromMap(testOptions.toMap()));
+
+ assertEquivalentEntries(input, merged);
+
+ boolean hasRewrittenManifest = false;
+ for (ManifestFileMeta meta : merged) {
+ if (inputManifestFileNames.contains(meta.fileName())) {
+ continue;
+ }
+
+ hasRewrittenManifest = true;
+ List<ManifestEntry> entries = manifestFile.read(meta.fileName(),
meta.fileSize());
+ for (int i = 1; i < entries.size(); i++) {
+ int prevPartition = entries.get(i - 1).partition().getInt(0);
+ int currPartition = entries.get(i).partition().getInt(0);
+ assertThat(currPartition)
+ .as("Entries within rewritten manifest should be
sorted by partition")
+ .isGreaterThanOrEqualTo(prevPartition);
+ }
+ }
+ assertThat(hasRewrittenManifest)
+ .as("Small max rewrite size should still rewrite at least one
useful batch")
+ .isTrue();
+ }
+
@Test
public void testManifestSortWithSpillableExternalSortBuffer() {
List<ManifestFileMeta> input = new ArrayList<>();
