This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4926d60aba [common] Preserve empty scored results during serialization
(#8924)
4926d60aba is described below
commit 4926d60aba22587f243f57126cf224e333ae3cda
Author: QuakeWang <[email protected]>
AuthorDate: Thu Jul 30 19:18:59 2026 +0800
[common] Preserve empty scored results during serialization (#8924)
---
.../globalindex/GlobalIndexResultSerializer.java | 58 ++++++++++++---
.../globalindex/GlobalIndexSerDeUtilsTest.java | 85 ++++++++++++++++++++++
2 files changed, 134 insertions(+), 9 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexResultSerializer.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexResultSerializer.java
index 5559c59c4f..56fdb0c68f 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexResultSerializer.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexResultSerializer.java
@@ -35,7 +35,11 @@ import static
org.apache.paimon.utils.Preconditions.checkArgument;
/** GlobalIndexResultSerializer to serialize and deserialize
GlobalIndexResult. */
public class GlobalIndexResultSerializer implements
Serializer<GlobalIndexResult> {
- private static final int VERSION = 1;
+ private static final int VERSION_1 = 1;
+ private static final int VERSION_2 = 2;
+
+ private static final byte UNSCORED_RESULT_KIND = 0;
+ private static final byte SCORED_RESULT_KIND = 1;
@Override
public Serializer<GlobalIndexResult> duplicate() {
@@ -59,7 +63,9 @@ public class GlobalIndexResultSerializer implements
Serializer<GlobalIndexResult
@Override
public void serialize(GlobalIndexResult globalIndexResult, DataOutputView
dataOutput)
throws IOException {
- dataOutput.writeInt(VERSION);
+ boolean isScored = globalIndexResult instanceof
ScoredGlobalIndexResult;
+ dataOutput.writeInt(VERSION_2);
+ dataOutput.writeByte(isScored ? SCORED_RESULT_KIND :
UNSCORED_RESULT_KIND);
RoaringNavigableMap64 roaringNavigableMap64 =
globalIndexResult.results();
byte[] bytes = roaringNavigableMap64.serialize();
@@ -67,7 +73,7 @@ public class GlobalIndexResultSerializer implements
Serializer<GlobalIndexResult
dataOutput.writeInt(bytes.length);
dataOutput.write(bytes);
- if (globalIndexResult instanceof ScoredGlobalIndexResult) {
+ if (isScored) {
ScoredGlobalIndexResult scored = (ScoredGlobalIndexResult)
globalIndexResult;
dataOutput.writeInt(roaringNavigableMap64.getIntCardinality());
ScoreGetter scoreGetter = scored.scoreGetter();
@@ -82,21 +88,55 @@ public class GlobalIndexResultSerializer implements
Serializer<GlobalIndexResult
@Override
public GlobalIndexResult deserialize(DataInputView dataInput) throws
IOException {
int version = dataInput.readInt();
- if (version != VERSION) {
- throw new IllegalStateException("Invalid version: " + version);
+ if (version == VERSION_1) {
+ return deserializeV1(dataInput);
+ }
+ if (version == VERSION_2) {
+ return deserializeV2(dataInput);
+ }
+ throw new IllegalStateException("Invalid version: " + version);
+ }
+
+ private GlobalIndexResult deserializeV1(DataInputView dataInput) throws
IOException {
+ RoaringNavigableMap64 roaringNavigableMap64 =
deserializeBitmap(dataInput);
+ int scoreSize = dataInput.readInt();
+ // V1 inferred the result kind from score count, so empty scored
results are
+ // indistinguishable.
+ if (scoreSize == 0) {
+ return GlobalIndexResult.create(roaringNavigableMap64);
+ }
+ return deserializeScoredResult(dataInput, roaringNavigableMap64,
scoreSize);
+ }
+
+ private GlobalIndexResult deserializeV2(DataInputView dataInput) throws
IOException {
+ byte resultKind = dataInput.readByte();
+ if (resultKind != UNSCORED_RESULT_KIND && resultKind !=
SCORED_RESULT_KIND) {
+ throw new IllegalStateException("Invalid result kind: " +
resultKind);
}
+ RoaringNavigableMap64 roaringNavigableMap64 =
deserializeBitmap(dataInput);
+ int scoreSize = dataInput.readInt();
+ if (resultKind == UNSCORED_RESULT_KIND) {
+ checkArgument(
+ scoreSize == 0, "Unexpected score size for unscored
result: %s", scoreSize);
+ return GlobalIndexResult.create(roaringNavigableMap64);
+ }
+ return deserializeScoredResult(dataInput, roaringNavigableMap64,
scoreSize);
+ }
+
+ private RoaringNavigableMap64 deserializeBitmap(DataInputView dataInput)
throws IOException {
int size = dataInput.readInt();
byte[] bytes = new byte[size];
dataInput.readFully(bytes);
RoaringNavigableMap64 roaringNavigableMap64 = new
RoaringNavigableMap64();
roaringNavigableMap64.deserialize(bytes);
- int scoreSize = dataInput.readInt();
+ return roaringNavigableMap64;
+ }
- if (scoreSize == 0) {
- return GlobalIndexResult.create(roaringNavigableMap64);
- }
+ private ScoredGlobalIndexResult deserializeScoredResult(
+ DataInputView dataInput, RoaringNavigableMap64
roaringNavigableMap64, int scoreSize)
+ throws IOException {
checkArgument(
scoreSize == roaringNavigableMap64.getIntCardinality(),
"Error size of score: "
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexSerDeUtilsTest.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexSerDeUtilsTest.java
index 7729620d29..148c64f324 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexSerDeUtilsTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexSerDeUtilsTest.java
@@ -33,6 +33,8 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link GlobalIndexResultSerializer}. */
public class GlobalIndexSerDeUtilsTest {
+ private static final int VERSION_1 = 1;
+
@Test
public void testSerializeAndDeserializeGlobalIndexResult() throws
IOException {
RoaringNavigableMap64 bitmap = RoaringNavigableMap64.bitmapOf(1, 5,
10, 100, 1000);
@@ -56,6 +58,32 @@ public class GlobalIndexSerDeUtilsTest {
assertThat(deserialized.results().isEmpty()).isTrue();
}
+ @Test
+ public void testSerializeAndDeserializeEmptyScoredGlobalIndexResult()
throws IOException {
+ ScoredGlobalIndexResult original =
ScoredGlobalIndexResult.createEmpty();
+
+ byte[] serialized = serialize(original);
+ GlobalIndexResult deserialized = deserialize(serialized);
+
+ assertThat(deserialized).isInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(deserialized.results().isEmpty()).isTrue();
+
assertThat(serialized).isNotEqualTo(serialize(GlobalIndexResult.createEmpty()));
+ }
+
+ @Test
+ public void testEmptyScoredGlobalIndexResultConvenienceMethods() throws
IOException {
+ GlobalIndexResultSerializer serializer = new
GlobalIndexResultSerializer();
+ ScoredGlobalIndexResult original =
ScoredGlobalIndexResult.createEmpty();
+
+ ScoredGlobalIndexResult deserialized =
+ serializer.deserialize(serializer.serialize(original));
+ GlobalIndexResult copied = serializer.copy(original);
+
+ assertThat(deserialized.results().isEmpty()).isTrue();
+ assertThat(copied).isInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(copied.results().isEmpty()).isTrue();
+ }
+
@Test
public void testSerializeAndDeserializeTopkGlobalIndexResult() throws
IOException {
RoaringNavigableMap64 bitmap = RoaringNavigableMap64.bitmapOf(1, 5,
10, 100);
@@ -106,6 +134,42 @@ public class GlobalIndexSerDeUtilsTest {
assertThat(scoreGetter.score(Long.MAX_VALUE - 1)).isEqualTo(0.1f);
}
+ @Test
+ public void testDeserializeV1GlobalIndexResult() throws IOException {
+ RoaringNavigableMap64 bitmap = RoaringNavigableMap64.bitmapOf(1, 5,
10);
+
+ GlobalIndexResult deserialized =
deserialize(serializeV1(GlobalIndexResult.create(bitmap)));
+
+
assertThat(deserialized).isNotInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(deserialized.results()).isEqualTo(bitmap);
+ }
+
+ @Test
+ public void testDeserializeV1ScoredGlobalIndexResult() throws IOException {
+ RoaringNavigableMap64 bitmap = RoaringNavigableMap64.bitmapOf(1, 5);
+ Map<Long, Float> scoreMap = new HashMap<>();
+ scoreMap.put(1L, 0.9f);
+ scoreMap.put(5L, 0.8f);
+
+ GlobalIndexResult deserialized =
+ deserialize(serializeV1(ScoredGlobalIndexResult.create(bitmap,
scoreMap::get)));
+
+ assertThat(deserialized).isInstanceOf(ScoredGlobalIndexResult.class);
+ ScoredGlobalIndexResult scored = (ScoredGlobalIndexResult)
deserialized;
+ assertThat(scored.results()).isEqualTo(bitmap);
+ assertThat(scored.scoreGetter().score(1L)).isEqualTo(0.9f);
+ assertThat(scored.scoreGetter().score(5L)).isEqualTo(0.8f);
+ }
+
+ @Test
+ public void testDeserializeV1EmptyScoredGlobalIndexResult() throws
IOException {
+ GlobalIndexResult deserialized =
+
deserialize(serializeV1(ScoredGlobalIndexResult.createEmpty()));
+
+
assertThat(deserialized).isNotInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(deserialized.results().isEmpty()).isTrue();
+ }
+
private byte[] serialize(GlobalIndexResult result) throws IOException {
GlobalIndexResultSerializer globalIndexResultSerializer = new
GlobalIndexResultSerializer();
DataOutputSerializer dataOutputSerializer = new
DataOutputSerializer(1024);
@@ -118,4 +182,25 @@ public class GlobalIndexSerDeUtilsTest {
DataInputDeserializer dataInputDeserializer = new
DataInputDeserializer(data);
return globalIndexResultSerializer.deserialize(dataInputDeserializer);
}
+
+ private byte[] serializeV1(GlobalIndexResult result) throws IOException {
+ DataOutputSerializer dataOutputSerializer = new
DataOutputSerializer(1024);
+ dataOutputSerializer.writeInt(VERSION_1);
+
+ RoaringNavigableMap64 bitmap = result.results();
+ byte[] bitmapBytes = bitmap.serialize();
+ dataOutputSerializer.writeInt(bitmapBytes.length);
+ dataOutputSerializer.write(bitmapBytes);
+
+ if (result instanceof ScoredGlobalIndexResult) {
+ dataOutputSerializer.writeInt(bitmap.getIntCardinality());
+ ScoreGetter scoreGetter = ((ScoredGlobalIndexResult)
result).scoreGetter();
+ for (long rowId : bitmap) {
+ dataOutputSerializer.writeFloat(scoreGetter.score(rowId));
+ }
+ } else {
+ dataOutputSerializer.writeInt(0);
+ }
+ return dataOutputSerializer.getCopyOfBuffer();
+ }
}
