This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4156dd3ee5 [core] Remove configurable manifest format (#9154)
4156dd3ee5 is described below
commit 4156dd3ee51110b1ea3bf1a703ff73e93d4554d6
Author: YeJunHao <[email protected]>
AuthorDate: Mon Aug 10 21:15:44 2026 +0800
[core] Remove configurable manifest format (#9154)
Manifest files are standardized on Avro, and the Python implementation
already supports Avro manifests only. Keeping `manifest.format` exposes
unsupported alternative formats and can make manifests unreadable across
implementations.
---
docs/generated/core_configuration.html | 6 ---
.../main/java/org/apache/paimon/CoreOptions.java | 10 ----
.../java/org/apache/paimon/format/FileFormat.java | 2 +-
.../java/org/apache/paimon/FileFormatTest.java | 10 ++++
.../test/java/org/apache/paimon/TestFileStore.java | 1 -
.../DataEvolutionRowIdReassignerTest.java | 55 ++--------------------
.../paimon/table/system/ManifestsTableTest.java | 4 +-
.../apache/paimon/flink/CatalogTableITCase.java | 10 ++--
8 files changed, 20 insertions(+), 78 deletions(-)
diff --git a/docs/generated/core_configuration.html
b/docs/generated/core_configuration.html
index 97aad92975..70989a7bc7 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -1035,12 +1035,6 @@ Mainly to resolve data skew on primary keys. We
recommend starting with 64 mb wh
<td>Boolean</td>
<td>For DELETE manifest entry in manifest file, drop stats to
reduce memory and storage. Default value is false only for compatibility of old
reader.</td>
</tr>
- <tr>
- <td><h5>manifest.format</h5></td>
- <td style="word-wrap: break-word;">"avro"</td>
- <td>String</td>
- <td>Specify the message format of manifest files.</td>
- </tr>
<tr>
<td><h5>manifest.full-compaction-threshold-size</h5></td>
<td style="word-wrap: break-word;">16 mb</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 3c644d953b..c65da81a80 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -509,12 +509,6 @@ public class CoreOptions implements Serializable {
+ "in the previous file. This must not
exceed "
+
"'variant.shredding.minFieldCardinalityRatio'.");
- public static final ConfigOption<String> MANIFEST_FORMAT =
- key("manifest.format")
- .stringType()
- .defaultValue(CoreOptions.FILE_FORMAT_AVRO)
- .withDescription("Specify the message format of manifest
files.");
-
public static final ConfigOption<String> MANIFEST_COMPRESSION =
key("manifest.compression")
.stringType()
@@ -3044,10 +3038,6 @@ public class CoreOptions implements Serializable {
return normalizeFileFormat(options.get(FILE_FORMAT));
}
- public String manifestFormatString() {
- return normalizeFileFormat(options.get(MANIFEST_FORMAT));
- }
-
public String manifestCompression() {
return options.get(MANIFEST_COMPRESSION);
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
index 430fc1404c..d6bfe7a0d3 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
@@ -117,6 +117,6 @@ public abstract class FileFormat {
}
public static FileFormat manifestFormat(CoreOptions options) {
- return FileFormat.fromIdentifier(options.manifestFormatString(),
options.toConfiguration());
+ return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO,
options.toConfiguration());
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
index 1981c13a0c..fd40448472 100644
--- a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
@@ -130,6 +130,16 @@ public class FileFormatTest {
assertThat(orcFileFormat.readBatchSize()).isEqualTo(1024);
}
+ @Test
+ public void testManifestFormatIsAlwaysAvro() {
+ Options tableOptions = new Options();
+ tableOptions.set(CoreOptions.FILE_FORMAT, "orc");
+
+ FileFormat manifestFormat = FileFormat.manifestFormat(new
CoreOptions(tableOptions));
+
+
assertThat(manifestFormat.getFormatIdentifier()).isEqualTo(CoreOptions.FILE_FORMAT_AVRO);
+ }
+
public FileFormat createFileFormat(String codec) {
Options tableOptions = new Options();
tableOptions.set(CoreOptions.FILE_FORMAT,
CoreOptions.FILE_FORMAT_AVRO);
diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
index 6f4a9aa89d..b670ffa48f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
@@ -819,7 +819,6 @@ public class TestFileStore extends KeyValueFileStore {
MemorySize.parse((ThreadLocalRandom.current().nextInt(16)
+ 1) + "kb"));
conf.set(CoreOptions.FILE_FORMAT, format);
- conf.set(CoreOptions.MANIFEST_FORMAT, format);
conf.set(CoreOptions.PATH, root);
conf.set(CoreOptions.BUCKET, numBuckets);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
index ee3bcac92b..0de855be14 100644
---
a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
@@ -133,7 +133,7 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
@Test
public void testProjectedPlannerBuildsExpectedPlan() throws Exception {
- catalog.createTable(identifier(), projectedPlannerSchema(null), true);
+ catalog.createTable(identifier(), projectedPlannerSchema(), true);
FileStoreTable table = getTableDefault();
writeOneRow(table, "a", 0);
writeOneRow(table, "b", 1);
@@ -155,7 +155,7 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
@Test
public void testProjectedPlannerAppliesPartitionPredicate() throws
Exception {
- catalog.createTable(identifier(), projectedPlannerSchema(null), true);
+ catalog.createTable(identifier(), projectedPlannerSchema(), true);
FileStoreTable table = getTableDefault();
writeOneRow(table, "a", 0);
writeOneRow(table, "b", 1);
@@ -173,23 +173,6 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
assertThat(result.totalOffset).isZero();
}
- @Test
- public void testProjectedPlannerReadsOrcManifest() throws Exception {
- catalog.createTable(identifier(), projectedPlannerSchema("orc"), true);
- FileStoreTable table = getTableDefault();
- writeOneRow(table, "a", 0);
- writeOneRow(table, "b", 1);
- writeOneRow(table, "a", 2);
-
- DataEvolutionRowIdAssignmentPlanner.Result result =
planProjectedState(table, null);
-
- assertThat(result.rowIdMappings).hasSize(1);
- RowRangeMappingIndex mapping =
result.rowIdMappings.values().iterator().next();
- assertThat(mapping.map(new Range(0L, 0L))).hasValue(new Range(0L, 0L));
- assertThat(mapping.map(new Range(2L, 2L))).hasValue(new Range(1L, 1L));
- assertThat(result.totalOffset).isEqualTo(2L);
- }
-
@Test
public void testInitialCapacityEstimatesLiveFileRangesAfterDeletes() {
assertThat(
@@ -217,7 +200,7 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
return state.plan(groups);
}
- private Schema projectedPlannerSchema(String manifestFormat) {
+ private Schema projectedPlannerSchema() {
Schema.Builder schemaBuilder = Schema.newBuilder();
schemaBuilder.column("pt", DataTypes.STRING());
schemaBuilder.column("id", DataTypes.INT());
@@ -225,9 +208,6 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
schemaBuilder.partitionKeys("pt");
schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
- if (manifestFormat != null) {
- schemaBuilder.option(CoreOptions.MANIFEST_FORMAT.key(),
manifestFormat);
- }
return schemaBuilder.build();
}
@@ -308,35 +288,6 @@ public class DataEvolutionRowIdReassignerTest extends
TableTestBase {
assertThat(compactPlan).isPresent();
}
- @Test
- public void testCompactAndLegacyPlansMatchForOrcManifests() throws
Exception {
- Schema base = schemaDefault();
- Map<String, String> options = new HashMap<>(base.options());
- options.put(CoreOptions.MANIFEST_FORMAT.key(), "orc");
- catalog.createTable(
- identifier(),
- new Schema(
- base.fields(),
- base.partitionKeys(),
- base.primaryKeys(),
- options,
- base.comment()),
- true);
- FileStoreTable table = getTableDefault();
- writeOneRow(table, "a", 0);
- writeOneRow(table, "b", 1);
- writeOneRow(table, "a", 2);
-
- Snapshot snapshot = table.snapshotManager().latestSnapshot();
- ManifestList manifestList =
table.store().manifestListFactory().create();
- List<ManifestFileMeta> manifestMetas =
manifestList.readDataManifests(snapshot);
-
- Optional<AssignmentPlanView> compactPlan = compactPlanView(table,
manifestMetas, null);
- Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table,
manifestMetas, null);
- assertPlanViewsEqual(compactPlan, legacyPlan);
- assertThat(compactPlan).isPresent();
- }
-
@Test
public void testCompactAndLegacyPlansMatchForSpecialFileRanges() throws
Exception {
createTableDefault();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java
index c8bf00e813..0d7a39c23d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java
@@ -90,9 +90,7 @@ public class ManifestsTableTest extends TableTestBase {
ManifestList.Factory factory =
new ManifestList.Factory(
fileIO,
- FileFormat.fromIdentifier(
-
CoreOptions.MANIFEST_FORMAT.defaultValue().toString(),
- new Options()),
+
FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, new Options()),
"zstd",
createNonPartFactory(tablePath),
null);
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
index 802bca58f5..3dc809c5ef 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
@@ -278,7 +278,7 @@ public class CatalogTableITCase extends CatalogITCaseBase {
sql("ALTER TABLE T SET ('snapshot.time-retained' = '5 h')");
sql("ALTER TABLE T SET ('snapshot.num-retained.max' = '20')");
sql("ALTER TABLE T SET ('snapshot.num-retained.min' = '18')");
- sql("ALTER TABLE T SET ('manifest.format' = 'avro')");
+ sql("ALTER TABLE T SET ('manifest.compression' = 'snappy')");
String actualResult = sql("SHOW CREATE TABLE T$schemas").toString();
String expectedResult =
@@ -315,8 +315,8 @@ public class CatalogTableITCase extends CatalogITCaseBase {
+
"{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], "
+
"{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5
h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"snapshot.num-retained.min\":\"18\"},
], "
+ "+I[4,
[{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT
NULL\"},{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}],
[], [\"a\"], "
- +
"{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5
h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"manifest.format\":\"avro\","
- + "\"snapshot.num-retained.min\":\"18\"}, ]]");
+ +
"{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5
h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"snapshot.num-retained.min\":\"18\","
+ + "\"manifest.compression\":\"snappy\"}, ]]");
result =
sql(
@@ -375,8 +375,8 @@ public class CatalogTableITCase extends CatalogITCaseBase {
+
"{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], "
+
"{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5
h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"snapshot.num-retained.min\":\"18\"},
], "
+ "+I[4,
[{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT
NULL\"},{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}],
[], [\"a\"], "
- +
"{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5
h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"manifest.format\":\"avro\","
- + "\"snapshot.num-retained.min\":\"18\"}, ]]");
+ +
"{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5
h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"snapshot.num-retained.min\":\"18\","
+ + "\"manifest.compression\":\"snappy\"}, ]]");
// check with not exist schema id
assertThatThrownBy(
