This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 142f8239be [core] Preserve index manifest compatibility (#9004)
142f8239be is described below
commit 142f8239be9334c9d7621bacdf8fb63840b9bee1
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Aug 3 21:22:39 2026 +0800
[core] Preserve index manifest compatibility (#9004)
---
.../manifest/IndexManifestEntrySerializer.java | 9 ++--
.../manifest/IndexManifestEntrySerializerTest.java | 33 ++++----------
.../manifest/IndexManifestFileHandlerTest.java | 51 ++++++++++++++++++++++
3 files changed, 63 insertions(+), 30 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
index d8649dab2f..d6979de802 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
@@ -43,7 +43,7 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
@Override
public int getVersion() {
- return 2;
+ return 1;
}
@Override
@@ -77,21 +77,20 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
@Override
public IndexManifestEntry convertFrom(int version, InternalRow row) {
- if (version < 1 || version > 2) {
+ if (version != 1) {
throw new UnsupportedOperationException("Unsupported version: " +
version);
}
GlobalIndexMeta globalIndexMeta = null;
if (!row.isNullAt(9)) {
- InternalRow globalIndexRow = row.getRow(9, version == 1 ? 5 : 6);
+ InternalRow globalIndexRow = row.getRow(9,
GlobalIndexMeta.SCHEMA.getFieldCount());
long rowRangeStart = globalIndexRow.getLong(0);
long rowRangeEnd = globalIndexRow.getLong(1);
int indexFieldId = globalIndexRow.getInt(2);
int[] extralFields =
globalIndexRow.isNullAt(3) ? null :
globalIndexRow.getArray(3).toIntArray();
byte[] indexMeta = globalIndexRow.isNullAt(4) ? null :
globalIndexRow.getBinary(4);
- byte[] sourceMeta =
- version == 1 || globalIndexRow.isNullAt(5) ? null :
globalIndexRow.getBinary(5);
+ byte[] sourceMeta = globalIndexRow.isNullAt(5) ? null :
globalIndexRow.getBinary(5);
globalIndexMeta =
new GlobalIndexMeta(
rowRangeStart,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
index d7cccdd880..28be88b809 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
@@ -20,20 +20,13 @@ package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.data.JoinedRow;
-import org.apache.paimon.data.serializer.InternalRowSerializer;
-import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.utils.ObjectSerializer;
import org.apache.paimon.utils.ObjectSerializerTestBase;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import org.junit.jupiter.api.Test;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Random;
@@ -45,7 +38,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<IndexManifestEntry> {
@Test
- void testReadsGlobalIndexWithoutSourceMeta() throws IOException {
+ void testReadsGlobalIndexWithoutSourceMeta() {
IndexManifestEntrySerializer serializer = new
IndexManifestEntrySerializer();
IndexManifestEntry entry =
new IndexManifestEntry(
@@ -60,25 +53,14 @@ public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<I
new GlobalIndexMeta(0, 9, 7, null, new byte[]
{1}),
null));
GenericRow serialized = (GenericRow) serializer.convertTo(entry);
- InternalRowSerializer legacyGlobalIndexSerializer =
- InternalSerializers.create(
- GlobalIndexMeta.SCHEMA.copy(
- GlobalIndexMeta.SCHEMA.getFields().subList(0,
5)));
- BinaryRow legacyGlobalIndexRow =
- legacyGlobalIndexSerializer
- .toBinaryRow(GenericRow.of(0L, 9L, 7, null, new byte[]
{1}))
- .copy();
- serialized.setField(9, legacyGlobalIndexRow);
-
- InternalRow version1Row = new JoinedRow().replace(GenericRow.of(1),
serialized);
- InternalRowSerializer versionedRowSerializer =
- InternalSerializers.create(
-
VersionedObjectSerializer.versionType(IndexManifestEntry.SCHEMA));
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- versionedRowSerializer.serialize(version1Row, new
DataOutputViewStreamWrapper(out));
+ assertThat(serialized.getRow(9,
GlobalIndexMeta.SCHEMA.getFieldCount()).getFieldCount())
+ .isEqualTo(6);
GlobalIndexMeta restored =
-
serializer.deserializeFromBytes(out.toByteArray()).indexFile().globalIndexMeta();
+ serializer
+ .convertFrom(serializer.getVersion(), serialized)
+ .indexFile()
+ .globalIndexMeta();
assertThat(restored.indexMeta()).containsExactly(1);
assertThat(restored.sourceMeta()).isNull();
@@ -87,6 +69,7 @@ public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<I
@Test
void testGlobalIndexSourceMetaRoundTrip() throws IOException {
IndexManifestEntrySerializer serializer = new
IndexManifestEntrySerializer();
+ assertThat(serializer.getVersion()).isEqualTo(1);
IndexManifestEntry entry =
new IndexManifestEntry(
FileKind.ADD,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
index cfcc896525..c022d05b72 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
@@ -20,19 +20,28 @@ package org.apache.paimon.manifest;
import org.apache.paimon.TestAppendFileStore;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.FormatReaderFactory;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.table.BucketMode;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.CloseableIterator;
+import org.apache.paimon.utils.VersionedObjectSerializer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import static
org.apache.paimon.index.IndexFileMetaSerializerTest.randomDeletionVectorIndexFile;
+import static org.apache.paimon.utils.FileUtils.createFormatReader;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -118,6 +127,48 @@ public class IndexManifestFileHandlerTest {
assertThat(entries.contains(entry4)).isTrue();
}
+ @Test
+ public void testNewIndexManifestReadableWithLegacySchema() throws
Exception {
+ TestAppendFileStore fileStore =
+ TestAppendFileStore.createAppendStore(tempDir, new
HashMap<>());
+ FileFormat fileFormat = FileFormat.manifestFormat(fileStore.options());
+ IndexManifestFile indexManifestFile =
+ new IndexManifestFile.Factory(
+ fileStore.fileIO(),
+ fileFormat,
+ "zstd",
+ fileStore.pathFactory(),
+ null)
+ .create();
+ IndexManifestFileHandler handler =
+ new IndexManifestFileHandler(indexManifestFile,
BucketMode.HASH_FIXED);
+
+ String manifestFile = handler.write(null,
Arrays.asList(pkVectorEntry("btree", "index")));
+
+ RowType legacyGlobalIndexSchema =
+
GlobalIndexMeta.SCHEMA.copy(GlobalIndexMeta.SCHEMA.getFields().subList(0, 5));
+ List<DataField> legacyEntryFields = new
ArrayList<>(IndexManifestEntry.SCHEMA.getFields());
+ legacyEntryFields.set(9,
legacyEntryFields.get(9).newType(legacyGlobalIndexSchema));
+ RowType legacySchema =
+ VersionedObjectSerializer.versionType(new RowType(false,
legacyEntryFields));
+ FormatReaderFactory legacyReaderFactory =
+ fileFormat.createReaderFactory(legacySchema, legacySchema, new
ArrayList<>());
+ Path path =
fileStore.pathFactory().indexManifestFileFactory().toPath(manifestFile);
+
+ try (CloseableIterator<InternalRow> iterator =
+ createFormatReader(fileStore.fileIO(), legacyReaderFactory,
path, null)
+ .toCloseableIterator()) {
+ InternalRow row = iterator.next();
+ assertThat(row.getInt(0)).isEqualTo(1);
+ InternalRow globalIndex = row.getRow(10, 5);
+ assertThat(globalIndex.getLong(0)).isEqualTo(0);
+ assertThat(globalIndex.getLong(1)).isEqualTo(1);
+ assertThat(globalIndex.getInt(2)).isEqualTo(1);
+ assertThat(globalIndex.isNullAt(4)).isTrue();
+ assertThat(iterator.hasNext()).isFalse();
+ }
+ }
+
@Test
public void testGlobalIndexOverlappingRangeRejectedWhenPreviousFileKept()
throws Exception {
TestAppendFileStore fileStore =
