This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new cdd3c2193f [core] Remove MultiValueIndexFileMeta (#9305)
cdd3c2193f is described below
commit cdd3c2193fc44ec6cb2aa9920d8b54c56e357a15
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Aug 19 19:30:51 2026 +0800
[core] Remove MultiValueIndexFileMeta (#9305)
---
.../bitmap/MultiValueBitmapIndexReader.java | 26 +-------
.../bitmap/MultiValueBitmapIndexWriter.java | 8 +--
.../bitmap/MultiValueGlobalIndexer.java | 4 +-
.../bitmap/MultiValueIndexFileMeta.java | 73 ----------------------
.../bitmap/MultiValueBitmapIndexReaderTest.java | 45 +------------
5 files changed, 6 insertions(+), 150 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReader.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReader.java
index f8856ba221..f76abc817a 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReader.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReader.java
@@ -24,8 +24,6 @@ import org.apache.paimon.globalindex.GlobalIndexResult;
import org.apache.paimon.globalindex.KeySerializer;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.predicate.FieldRef;
-import org.apache.paimon.types.ArrayType;
-import org.apache.paimon.types.DataType;
import java.io.IOException;
import java.util.ArrayList;
@@ -39,26 +37,16 @@ import java.util.concurrent.ExecutorService;
/** Exposes array-element membership over the bitmap global index format. */
public class MultiValueBitmapIndexReader implements GlobalIndexReader {
- private final DataType elementType;
- private final boolean compatibleElementType;
private final KeySerializer keySerializer;
private final LazyFilteredBitmapReader bitmapReader;
MultiValueBitmapIndexReader(
GlobalIndexFileReader fileReader,
List<GlobalIndexIOMeta> files,
- DataType elementType,
KeySerializer keySerializer,
long totalRowCount,
ExecutorService executor) {
- this.elementType = elementType;
this.keySerializer = keySerializer;
- this.compatibleElementType =
- files.stream()
- .allMatch(
- file ->
-
MultiValueIndexFileMeta.hasCompatibleElementType(
- file.metadata(), elementType));
this.bitmapReader =
new LazyFilteredBitmapReader(
fileReader, files, keySerializer, 0, totalRowCount,
executor);
@@ -77,25 +65,19 @@ public class MultiValueBitmapIndexReader implements
GlobalIndexReader {
@Override
public CompletableFuture<Optional<GlobalIndexResult>> visitArrayContains(
FieldRef fieldRef, Object literal) {
- if (!supports(fieldRef)) {
- return unsupported();
- }
return bitmapReader.visitEqual(fieldRef, literal);
}
@Override
public CompletableFuture<Optional<GlobalIndexResult>> visitArraysOverlap(
FieldRef fieldRef, List<Object> literals) {
- if (!supports(fieldRef)) {
- return unsupported();
- }
return bitmapReader.visitIn(fieldRef, literals);
}
@Override
public CompletableFuture<Optional<GlobalIndexResult>>
visitArrayContainsAll(
FieldRef fieldRef, List<Object> literals) {
- if (!supports(fieldRef) || literals.isEmpty()) {
+ if (literals.isEmpty()) {
return unsupported();
}
@@ -225,10 +207,4 @@ public class MultiValueBitmapIndexReader implements
GlobalIndexReader {
private static CompletableFuture<Optional<GlobalIndexResult>>
unsupported() {
return CompletableFuture.completedFuture(Optional.empty());
}
-
- private boolean supports(FieldRef fieldRef) {
- return compatibleElementType
- && fieldRef.type() instanceof ArrayType
- && ((ArrayType)
fieldRef.type()).getElementType().equals(elementType);
- }
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java
index f2c9982220..beda6cfaba 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java
@@ -26,7 +26,6 @@ import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.SortedIndexFileMeta;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
import org.apache.paimon.memory.MemorySlice;
-import org.apache.paimon.types.DataType;
import org.apache.paimon.utils.Preconditions;
import org.apache.paimon.utils.RoaringNavigableMap64;
@@ -42,7 +41,6 @@ import java.util.List;
public class MultiValueBitmapIndexWriter implements
GlobalIndexSingleColumnWriter, Closeable {
private final GlobalIndexFileWriter fileWriter;
- private final DataType elementType;
private final KeySerializer keySerializer;
private final Comparator<Object> comparator;
private final int dictionaryBlockSize;
@@ -59,12 +57,10 @@ public class MultiValueBitmapIndexWriter implements
GlobalIndexSingleColumnWrite
MultiValueBitmapIndexWriter(
GlobalIndexFileWriter fileWriter,
- DataType elementType,
KeySerializer keySerializer,
int dictionaryBlockSize,
@Nullable BlockCompressionFactory compressionFactory) {
this.fileWriter = fileWriter;
- this.elementType = elementType;
this.keySerializer = keySerializer;
this.comparator = keySerializer.createComparator();
this.dictionaryBlockSize = dictionaryBlockSize;
@@ -125,9 +121,7 @@ public class MultiValueBitmapIndexWriter implements
GlobalIndexSingleColumnWrite
throw new RuntimeException("Error in closing multivalue index
writer.", e);
}
- byte[] meta =
- MultiValueIndexFileMeta.serialize(
- new SortedIndexFileMeta(firstKey, lastKey, false),
elementType);
+ byte[] meta = new SortedIndexFileMeta(firstKey, lastKey,
false).serialize();
return Collections.singletonList(new ResultEntry(fileName,
sourceRowCount, meta));
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueGlobalIndexer.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueGlobalIndexer.java
index e377206324..5f59e015f0 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueGlobalIndexer.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueGlobalIndexer.java
@@ -76,7 +76,7 @@ public class MultiValueGlobalIndexer implements
SortedGlobalIndexer {
public MultiValueBitmapIndexWriter createWriter(GlobalIndexFileWriter
fileWriter)
throws IOException {
return new MultiValueBitmapIndexWriter(
- fileWriter, elementType, keySerializer, dictionaryBlockSize,
compressionFactory);
+ fileWriter, keySerializer, dictionaryBlockSize,
compressionFactory);
}
@Override
@@ -86,7 +86,7 @@ public class MultiValueGlobalIndexer implements
SortedGlobalIndexer {
long totalRowCount,
ExecutorService executor) {
return new MultiValueBitmapIndexReader(
- fileReader, files, elementType, keySerializer, totalRowCount,
executor);
+ fileReader, files, keySerializer, totalRowCount, executor);
}
private static class ArrayElementKeyExtractor implements
GlobalIndexKeyExtractor {
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueIndexFileMeta.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueIndexFileMeta.java
deleted file mode 100644
index 08b4b45ad8..0000000000
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueIndexFileMeta.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.globalindex.bitmap;
-
-import org.apache.paimon.globalindex.SortedIndexFileMeta;
-import org.apache.paimon.types.DataType;
-import org.apache.paimon.utils.JsonSerdeUtil;
-
-import javax.annotation.Nullable;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-
-/** Manifest-level sorted metadata with the element type used by a Multivalue
index. */
-public final class MultiValueIndexFileMeta {
-
- // "MVIM". The trailer leaves the SortedIndexFileMeta prefix readable by
older selectors.
- private static final int MAGIC = 0x4D56494D;
- private static final int TRAILER_SIZE = Integer.BYTES * 2;
-
- private MultiValueIndexFileMeta() {}
-
- public static byte[] serialize(SortedIndexFileMeta sortedMeta, DataType
elementType) {
- byte[] sortedBytes = sortedMeta.serialize();
- byte[] typeBytes = typeSignature(elementType);
- ByteBuffer buffer =
- ByteBuffer.allocate(sortedBytes.length + typeBytes.length +
TRAILER_SIZE);
- buffer.put(sortedBytes);
- buffer.put(typeBytes);
- buffer.putInt(typeBytes.length);
- buffer.putInt(MAGIC);
- return buffer.array();
- }
-
- public static boolean hasCompatibleElementType(
- @Nullable byte[] metadata, DataType elementType) {
- if (metadata == null || metadata.length < TRAILER_SIZE) {
- return false;
- }
- ByteBuffer trailer =
- ByteBuffer.wrap(metadata, metadata.length - TRAILER_SIZE,
TRAILER_SIZE);
- int typeLength = trailer.getInt();
- int magic = trailer.getInt();
- if (magic != MAGIC || typeLength < 0 || typeLength > metadata.length -
TRAILER_SIZE) {
- return false;
- }
- int typeOffset = metadata.length - TRAILER_SIZE - typeLength;
- return Arrays.equals(
- Arrays.copyOfRange(metadata, typeOffset, typeOffset +
typeLength),
- typeSignature(elementType));
- }
-
- private static byte[] typeSignature(DataType elementType) {
- return
JsonSerdeUtil.toJson(elementType).getBytes(StandardCharsets.UTF_8);
- }
-}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java
index 118d1d1f7c..eab2a26162 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java
@@ -109,14 +109,8 @@ class MultiValueBitmapIndexReaderTest {
Path path = new Path(basePath, result.fileName());
GlobalIndexIOMeta meta =
new GlobalIndexIOMeta(path, fileIO.getFileSize(path),
result.meta());
- assertThat(
- MultiValueIndexFileMeta.hasCompatibleElementType(
- result.meta(), DataTypes.STRING()))
- .isTrue();
- assertThat(
- MultiValueIndexFileMeta.hasCompatibleElementType(
- result.meta(), DataTypes.BIGINT()))
- .isFalse();
+ assertThat(result.meta())
+
.isEqualTo(SortedIndexFileMeta.deserialize(result.meta()).serialize());
try (GlobalIndexReader reader =
globalIndexer.createReader(
@@ -154,47 +148,12 @@ class MultiValueBitmapIndexReaderTest {
.join());
assertThat(reader.visitArrayContainsAll(fieldRef,
Collections.emptyList()).join())
.isEmpty();
- assertThat(
- reader.visitArrayContains(
- new FieldRef(
- 1, "tags",
DataTypes.ARRAY(DataTypes.BIGINT())),
- 1L)
- .join())
- .isEmpty();
- assertThat(
- reader.visitArraysOverlap(
- new FieldRef(
- 1, "tags",
DataTypes.ARRAY(DataTypes.BIGINT())),
- Collections.singletonList(1L))
- .join())
- .isEmpty();
- assertThat(
- reader.visitArrayContainsAll(
- new FieldRef(
- 1, "tags",
DataTypes.ARRAY(DataTypes.BIGINT())),
- Collections.singletonList(1L))
- .join())
- .isEmpty();
assertThat(reader.visitIsNull(fieldRef).join()).isEmpty();
assertThat(reader.visitIsNotNull(fieldRef).join()).isEmpty();
assertThat(reader.visitEqual(fieldRef,
array("A")).join()).isEmpty();
assertThat(reader.visitContains(fieldRef,
str("A")).join()).isEmpty();
}
-
- GlobalIndexIOMeta legacyMeta =
- new GlobalIndexIOMeta(
- path,
- fileIO.getFileSize(path),
-
SortedIndexFileMeta.deserialize(result.meta()).serialize());
- try (GlobalIndexReader reader =
- globalIndexer.createReader(
- fileReader,
- Collections.singletonList(legacyMeta),
- 5,
- newDirectExecutorService())) {
- assertThat(reader.visitArrayContains(fieldRef,
str("A")).join()).isEmpty();
- }
}
@Test
