This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch release-2.0
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/release-2.0 by this push:
new eb9b6dd694 [core] Fix index source metadata compatibility (#8952)
eb9b6dd694 is described below
commit eb9b6dd694c36fef4454acb20c38958305f3f15a
Author: Jingsong Lee <[email protected]>
AuthorDate: Fri Jul 31 12:18:10 2026 +0800
[core] Fix index source metadata compatibility (#8952)
---
.../org/apache/paimon/index/GlobalIndexMeta.java | 27 +++++
.../org/apache/paimon/index/IndexFileMeta.java | 6 +-
.../paimon/index/IndexFileMetaSerializer.java | 5 +-
.../paimon/index/IndexFileMetaV4Deserializer.java | 111 +++++++++++++++++++++
.../manifest/IndexManifestEntrySerializer.java | 10 +-
.../paimon/table/sink/CommitMessageSerializer.java | 11 +-
.../paimon/index/IndexFileMetaSerializerTest.java | 22 ++++
.../manifest/IndexManifestEntrySerializerTest.java | 58 +++++++++--
...festCommittableSerializerCompatibilityTest.java | 65 +++++++++---
9 files changed, 278 insertions(+), 37 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java
b/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java
index 727a6536e3..354c2261cf 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexMeta.java
@@ -30,6 +30,7 @@ import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
/** Schema for global index. */
public class GlobalIndexMeta {
@@ -153,4 +154,30 @@ public class GlobalIndexMeta {
}
return names;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ GlobalIndexMeta that = (GlobalIndexMeta) o;
+ return rowRangeStart == that.rowRangeStart
+ && rowRangeEnd == that.rowRangeEnd
+ && indexFieldId == that.indexFieldId
+ && Arrays.equals(extraFieldIds, that.extraFieldIds)
+ && Arrays.equals(indexMeta, that.indexMeta)
+ && Arrays.equals(sourceMeta, that.sourceMeta);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(rowRangeStart, rowRangeEnd, indexFieldId);
+ result = 31 * result + Arrays.hashCode(extraFieldIds);
+ result = 31 * result + Arrays.hashCode(indexMeta);
+ result = 31 * result + Arrays.hashCode(sourceMeta);
+ return result;
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
index a7c257a46e..cc77e30438 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
@@ -152,12 +152,14 @@ public class IndexFileMeta {
&& fileSize == that.fileSize
&& rowCount == that.rowCount
&& Objects.equals(dvRanges, that.dvRanges)
- && Objects.equals(externalPath, that.externalPath);
+ && Objects.equals(externalPath, that.externalPath)
+ && Objects.equals(globalIndexMeta, that.globalIndexMeta);
}
@Override
public int hashCode() {
- return Objects.hash(indexType, fileName, fileSize, rowCount, dvRanges,
externalPath);
+ return Objects.hash(
+ indexType, fileName, fileSize, rowCount, dvRanges,
externalPath, globalIndexMeta);
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
index a2acfd0ffc..ffaff2d0ad 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
@@ -73,10 +73,7 @@ public class IndexFileMetaSerializer extends
ObjectSerializer<IndexFileMeta> {
int[] extralFields =
globalIndexRow.isNullAt(3) ? null :
globalIndexRow.getArray(3).toIntArray();
byte[] indexMeta = globalIndexRow.isNullAt(4) ? null :
globalIndexRow.getBinary(4);
- byte[] sourceMeta =
- globalIndexRow.getFieldCount() <= 5 ||
globalIndexRow.isNullAt(5)
- ? null
- : globalIndexRow.getBinary(5);
+ byte[] sourceMeta = globalIndexRow.isNullAt(5) ? null :
globalIndexRow.getBinary(5);
globalIndexMeta =
new GlobalIndexMeta(
rowRangeStart,
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV4Deserializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV4Deserializer.java
new file mode 100644
index 0000000000..4ce5535a2f
--- /dev/null
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV4Deserializer.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.index;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.data.serializer.InternalSerializers;
+import org.apache.paimon.io.DataInputView;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.BigIntType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.RowType;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static
org.apache.paimon.index.IndexFileMetaSerializer.rowArrayDataToDvMetas;
+import static org.apache.paimon.utils.SerializationUtils.newStringType;
+
+/** Deserializer for {@link IndexFileMeta} in commit message version 11. */
+public class IndexFileMetaV4Deserializer implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final RowType GLOBAL_INDEX_SCHEMA =
+ new RowType(
+ true,
+ Arrays.asList(
+ new DataField(0, "_ROW_RANGE_START", new
BigIntType(false)),
+ new DataField(1, "_ROW_RANGE_END", new
BigIntType(false)),
+ new DataField(2, "_INDEX_FIELD_ID", new
IntType(false)),
+ new DataField(
+ 3, "_EXTRA_FIELD_IDS", DataTypes.ARRAY(new
IntType(false))),
+ new DataField(4, "_INDEX_META",
DataTypes.BYTES())));
+
+ public static final RowType SCHEMA =
+ new RowType(
+ false,
+ Arrays.asList(
+ new DataField(0, "_INDEX_TYPE",
newStringType(false)),
+ new DataField(1, "_FILE_NAME",
newStringType(false)),
+ new DataField(2, "_FILE_SIZE", new
BigIntType(false)),
+ new DataField(3, "_ROW_COUNT", new
BigIntType(false)),
+ new DataField(
+ 4,
+ "_DELETIONS_VECTORS_RANGES",
+ new ArrayType(true,
DeletionVectorMeta.SCHEMA)),
+ new DataField(5, "_EXTERNAL_PATH",
newStringType(true)),
+ new DataField(6, "_GLOBAL_INDEX",
GLOBAL_INDEX_SCHEMA)));
+
+ private final InternalRowSerializer rowSerializer;
+
+ public IndexFileMetaV4Deserializer() {
+ this.rowSerializer = InternalSerializers.create(SCHEMA);
+ }
+
+ private IndexFileMeta fromRow(InternalRow row) {
+ GlobalIndexMeta globalIndexMeta = null;
+ if (!row.isNullAt(6)) {
+ InternalRow globalIndexRow = row.getRow(6,
GLOBAL_INDEX_SCHEMA.getFieldCount());
+ long rowRangeStart = globalIndexRow.getLong(0);
+ long rowRangeEnd = globalIndexRow.getLong(1);
+ int indexFieldId = globalIndexRow.getInt(2);
+ int[] extraFields =
+ globalIndexRow.isNullAt(3) ? null :
globalIndexRow.getArray(3).toIntArray();
+ byte[] indexMeta = globalIndexRow.isNullAt(4) ? null :
globalIndexRow.getBinary(4);
+ globalIndexMeta =
+ new GlobalIndexMeta(
+ rowRangeStart, rowRangeEnd, indexFieldId,
extraFields, indexMeta);
+ }
+
+ return new IndexFileMeta(
+ row.getString(0).toString(),
+ row.getString(1).toString(),
+ row.getLong(2),
+ row.getLong(3),
+ row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)),
+ row.isNullAt(5) ? null : row.getString(5).toString(),
+ globalIndexMeta);
+ }
+
+ public List<IndexFileMeta> deserializeList(DataInputView source) throws
IOException {
+ int size = source.readInt();
+ List<IndexFileMeta> records = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ records.add(fromRow(rowSerializer.deserialize(source)));
+ }
+ return records;
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
index ef0b68d142..d8649dab2f 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
@@ -43,7 +43,7 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
@Override
public int getVersion() {
- return 1;
+ return 2;
}
@Override
@@ -77,13 +77,13 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
@Override
public IndexManifestEntry convertFrom(int version, InternalRow row) {
- if (version != 1) {
+ if (version < 1 || version > 2) {
throw new UnsupportedOperationException("Unsupported version: " +
version);
}
GlobalIndexMeta globalIndexMeta = null;
if (!row.isNullAt(9)) {
- InternalRow globalIndexRow = row.getRow(9, 6);
+ InternalRow globalIndexRow = row.getRow(9, version == 1 ? 5 : 6);
long rowRangeStart = globalIndexRow.getLong(0);
long rowRangeEnd = globalIndexRow.getLong(1);
int indexFieldId = globalIndexRow.getInt(2);
@@ -91,9 +91,7 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
globalIndexRow.isNullAt(3) ? null :
globalIndexRow.getArray(3).toIntArray();
byte[] indexMeta = globalIndexRow.isNullAt(4) ? null :
globalIndexRow.getBinary(4);
byte[] sourceMeta =
- globalIndexRow.getFieldCount() <= 5 ||
globalIndexRow.isNullAt(5)
- ? null
- : globalIndexRow.getBinary(5);
+ version == 1 || globalIndexRow.isNullAt(5) ? null :
globalIndexRow.getBinary(5);
globalIndexMeta =
new GlobalIndexMeta(
rowRangeStart,
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
index 5e621a9f78..2593a2a68a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
@@ -25,6 +25,7 @@ import org.apache.paimon.index.IndexFileMetaSerializer;
import org.apache.paimon.index.IndexFileMetaV1Deserializer;
import org.apache.paimon.index.IndexFileMetaV2Deserializer;
import org.apache.paimon.index.IndexFileMetaV3Deserializer;
+import org.apache.paimon.index.IndexFileMetaV4Deserializer;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFileMeta08Serializer;
@@ -51,7 +52,7 @@ import static
org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
/** {@link VersionedSerializer} for {@link CommitMessage}. */
public class CommitMessageSerializer implements
VersionedSerializer<CommitMessage> {
- public static final int CURRENT_VERSION = 11;
+ public static final int CURRENT_VERSION = 12;
private final DataFileMetaSerializer dataFileSerializer;
private final IndexFileMetaSerializer indexEntrySerializer;
@@ -64,6 +65,7 @@ public class CommitMessageSerializer implements
VersionedSerializer<CommitMessag
private IndexFileMetaV1Deserializer indexEntryV1Deserializer;
private IndexFileMetaV2Deserializer indexEntryV2Deserializer;
private IndexFileMetaV3Deserializer indexEntryV3Deserializer;
+ private IndexFileMetaV4Deserializer indexEntryV4Deserializer;
public CommitMessageSerializer() {
this.dataFileSerializer = new DataFileMetaSerializer();
@@ -217,8 +219,13 @@ public class CommitMessageSerializer implements
VersionedSerializer<CommitMessag
private IOExceptionSupplier<List<IndexFileMeta>> indexEntryDeserializer(
int version, DataInputView view) {
- if (version >= 11) {
+ if (version >= 12) {
return () -> indexEntrySerializer.deserializeList(view);
+ } else if (version == 11) {
+ if (indexEntryV4Deserializer == null) {
+ indexEntryV4Deserializer = new IndexFileMetaV4Deserializer();
+ }
+ return () -> indexEntryV4Deserializer.deserializeList(view);
} else if (version >= 9) {
if (indexEntryV3Deserializer == null) {
indexEntryV3Deserializer = new IndexFileMetaV3Deserializer();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
index 00d97d5435..33373741c6 100644
---
a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
@@ -51,6 +51,28 @@ public class IndexFileMetaSerializerTest extends
ObjectSerializerTestBase<IndexF
assertThat(restored.indexMeta()).containsExactly(3, 4);
}
+ @Test
+ void testEqualityIncludesGlobalIndexMeta() {
+ IndexFileMeta first =
+ globalIndexFile(
+ new GlobalIndexMeta(
+ 0, 9, 7, new int[] {8}, new byte[] {3}, new
byte[] {1}));
+ IndexFileMeta equal =
+ globalIndexFile(
+ new GlobalIndexMeta(
+ 0, 9, 7, new int[] {8}, new byte[] {3}, new
byte[] {1}));
+ IndexFileMeta different =
+ globalIndexFile(
+ new GlobalIndexMeta(
+ 0, 9, 7, new int[] {8}, new byte[] {3}, new
byte[] {2}));
+
+
assertThat(first).isEqualTo(equal).hasSameHashCodeAs(equal).isNotEqualTo(different);
+ }
+
+ private static IndexFileMeta globalIndexFile(GlobalIndexMeta
globalIndexMeta) {
+ return new IndexFileMeta("ivf-pq", "index-file", 100, 10,
globalIndexMeta, null);
+ }
+
@Override
protected ObjectSerializer<IndexFileMeta> serializer() {
return new IndexFileMetaSerializer();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
index 0678ed787a..d7cccdd880 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
@@ -21,13 +21,20 @@ package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.utils.ObjectSerializer;
import org.apache.paimon.utils.ObjectSerializerTestBase;
+import org.apache.paimon.utils.VersionedObjectSerializer;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.util.Random;
import static
org.apache.paimon.index.IndexFileMetaSerializerTest.randomIndexFile;
@@ -38,7 +45,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<IndexManifestEntry> {
@Test
- void testReadsGlobalIndexWithoutSourceMeta() {
+ void testReadsGlobalIndexWithoutSourceMeta() throws IOException {
IndexManifestEntrySerializer serializer = new
IndexManifestEntrySerializer();
IndexManifestEntry entry =
new IndexManifestEntry(
@@ -53,18 +60,55 @@ public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<I
new GlobalIndexMeta(0, 9, 7, null, new byte[]
{1}),
null));
GenericRow serialized = (GenericRow) serializer.convertTo(entry);
- serialized.setField(9, GenericRow.of(0L, 9L, 7, null, new byte[] {1}));
+ InternalRowSerializer legacyGlobalIndexSerializer =
+ InternalSerializers.create(
+ GlobalIndexMeta.SCHEMA.copy(
+ GlobalIndexMeta.SCHEMA.getFields().subList(0,
5)));
+ BinaryRow legacyGlobalIndexRow =
+ legacyGlobalIndexSerializer
+ .toBinaryRow(GenericRow.of(0L, 9L, 7, null, new byte[]
{1}))
+ .copy();
+ serialized.setField(9, legacyGlobalIndexRow);
+
+ InternalRow version1Row = new JoinedRow().replace(GenericRow.of(1),
serialized);
+ InternalRowSerializer versionedRowSerializer =
+ InternalSerializers.create(
+
VersionedObjectSerializer.versionType(IndexManifestEntry.SCHEMA));
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ versionedRowSerializer.serialize(version1Row, new
DataOutputViewStreamWrapper(out));
+
+ GlobalIndexMeta restored =
+
serializer.deserializeFromBytes(out.toByteArray()).indexFile().globalIndexMeta();
+
+ assertThat(restored.indexMeta()).containsExactly(1);
+ assertThat(restored.sourceMeta()).isNull();
+ }
+
+ @Test
+ void testGlobalIndexSourceMetaRoundTrip() throws IOException {
+ IndexManifestEntrySerializer serializer = new
IndexManifestEntrySerializer();
+ IndexManifestEntry entry =
+ new IndexManifestEntry(
+ FileKind.ADD,
+ BinaryRow.EMPTY_ROW,
+ 0,
+ new IndexFileMeta(
+ "ivf-pq",
+ "index-file",
+ 100,
+ 10,
+ new GlobalIndexMeta(
+ 0, 9, 7, null, new byte[] {3, 4}, new
byte[] {1, 2}),
+ null));
- InternalRow globalIndexRow = serialized.getRow(9, 5);
- assertThat(globalIndexRow.getFieldCount()).isEqualTo(5);
GlobalIndexMeta restored =
serializer
- .convertFrom(serializer.getVersion(), serialized)
+
.deserializeFromBytes(serializer.serializeToBytes(entry))
.indexFile()
.globalIndexMeta();
- assertThat(restored.indexMeta()).containsExactly(1);
- assertThat(restored.sourceMeta()).isNull();
+ assertThat(restored.indexMeta()).containsExactly(3, 4);
+ assertThat(restored.sourceMeta()).containsExactly(1, 2);
}
@Override
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
index c3a6ef9649..20eac11da7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
@@ -83,7 +83,8 @@ public class ManifestCommittableSerializerCompatibilityTest {
Arrays.asList("asdf", "qwer", "zxcv"));
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
GlobalIndexMeta globalIndexMeta =
- new GlobalIndexMeta(1L, 2L, 3, new int[] {5, 6, 7}, new byte[]
{0x23, 0x45});
+ new GlobalIndexMeta(
+ 1L, 2L, 3, new int[] {5, 6, 7}, new byte[] {0x23,
0x45}, new byte[] {0x67});
IndexFileMeta hashIndexFile =
new IndexFileMeta(
"my_index_type",
@@ -106,6 +107,52 @@ public class
ManifestCommittableSerializerCompatibilityTest {
dvRanges,
"external_path");
+ ManifestCommittable manifestCommittable =
+ createManifestCommittable(dataFiles, hashIndexFile,
devIndexFile);
+
+ ManifestCommittableSerializer serializer = new
ManifestCommittableSerializer();
+ byte[] bytes = serializer.serialize(manifestCommittable);
+ ManifestCommittable deserialized =
serializer.deserialize(serializer.getVersion(), bytes);
+ assertThat(deserialized).isEqualTo(manifestCommittable);
+ GlobalIndexMeta deserializedGlobalIndexMeta =
+ ((CommitMessageImpl) deserialized.fileCommittables().get(0))
+ .newFilesIncrement()
+ .newIndexFiles()
+ .get(0)
+ .globalIndexMeta();
+
assertThat(deserializedGlobalIndexMeta.sourceMeta()).containsExactly(0x67);
+
+ byte[] oldBytes =
+ IOUtils.readFully(
+ ManifestCommittableSerializerCompatibilityTest.class
+ .getClassLoader()
+ .getResourceAsStream("compatibility/" +
fileName),
+ true);
+ deserialized = serializer.deserialize(5, oldBytes);
+ GlobalIndexMeta legacyGlobalIndexMeta =
+ new GlobalIndexMeta(1L, 2L, 3, new int[] {5, 6, 7}, new byte[]
{0x23, 0x45});
+ IndexFileMeta legacyHashIndexFile =
+ new IndexFileMeta(
+ "my_index_type",
+ "my_index_file",
+ 1024 * 100,
+ 1002,
+ null,
+ null,
+ legacyGlobalIndexMeta);
+ assertThat(deserialized)
+ .isEqualTo(createManifestCommittable(dataFiles,
legacyHashIndexFile, devIndexFile));
+ deserializedGlobalIndexMeta =
+ ((CommitMessageImpl) deserialized.fileCommittables().get(0))
+ .newFilesIncrement()
+ .newIndexFiles()
+ .get(0)
+ .globalIndexMeta();
+ assertThat(deserializedGlobalIndexMeta.sourceMeta()).isNull();
+ }
+
+ private static ManifestCommittable createManifestCommittable(
+ List<DataFileMeta> dataFiles, IndexFileMeta hashIndexFile,
IndexFileMeta devIndexFile) {
CommitMessageImpl commitMessage =
new CommitMessageImpl(
singleColumn("my_partition"),
@@ -123,25 +170,11 @@ public class
ManifestCommittableSerializerCompatibilityTest {
dataFiles,
Collections.singletonList(devIndexFile),
Collections.emptyList()));
-
ManifestCommittable manifestCommittable =
new ManifestCommittable(5, 202020L,
Collections.singletonList(commitMessage));
manifestCommittable.addProperty("k1", "v1");
manifestCommittable.addProperty("k2", "v2");
-
- ManifestCommittableSerializer serializer = new
ManifestCommittableSerializer();
- byte[] bytes = serializer.serialize(manifestCommittable);
- ManifestCommittable deserialized =
serializer.deserialize(serializer.getVersion(), bytes);
- assertThat(deserialized).isEqualTo(manifestCommittable);
-
- byte[] oldBytes =
- IOUtils.readFully(
- ManifestCommittableSerializerCompatibilityTest.class
- .getClassLoader()
- .getResourceAsStream("compatibility/" +
fileName),
- true);
- deserialized = serializer.deserialize(5, oldBytes);
- assertThat(deserialized).isEqualTo(manifestCommittable);
+ return manifestCommittable;
}
@Test
