This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e2346aa337 [codegen] Fix map equaliser for keys without value-based
Java equality (#9738)
e2346aa337 is described below
commit e2346aa337249c59b1b05ee61d9ba6ddb99b252c
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Sep 11 16:05:27 2026 +0900
[codegen] Fix map equaliser for keys without value-based Java equality
(#9738)
---
.../apache/paimon/codegen/ScalarOperatorGens.scala | 15 +-
.../paimon/codegen/EqualiserCodeGeneratorTest.java | 175 +++++++++++++++++++++
2 files changed, 189 insertions(+), 1 deletion(-)
diff --git
a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
index d2cf7362ba..fa46e3d02a 100644
---
a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
+++
b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
@@ -337,7 +337,7 @@ object ScalarOperatorGens {
val mapDataUtil = className[InternalMapSerializer]
val stmt =
- if (containsFloatingPoint(keyType)) {
+ if (requiresElementWiseKeyMatch(keyType)) {
val leftKeyArrayTerm = newName("leftKeyArray")
val rightKeyArrayTerm = newName("rightKeyArray")
val leftValueArrayTerm = newName("leftValueArray")
@@ -472,5 +472,18 @@ object ScalarOperatorGens {
case _ => false
}
+ /**
+ * Whether map keys of the given type must be matched pairwise with the
generated key equality
+ * instead of being looked up through a [[java.util.Map]].
+ *
+ * A [[java.util.Map]] lookup is only correct when the internal
representation of the key has
+ * value-based `equals`/`hashCode`, which holds for numeric, character
string, decimal and
+ * temporal keys. Binary keys are `byte[]` and compare by identity;
composite keys may be columnar
+ * views without `hashCode` support or mix generic and binary
representations across the two maps;
+ * floating-point keys need `Float.compare`/`Double.compare` semantics.
+ */
+ private def requiresElementWiseKeyMatch(keyType: DataType): Boolean =
+ containsFloatingPoint(keyType) || isBinaryString(keyType) ||
!isComparable(keyType)
+
//
----------------------------------------------------------------------------------------------
}
diff --git
a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
index 3d75cdcf41..ab4ad74af8 100644
---
a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
+++
b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
@@ -50,6 +50,7 @@ import org.junit.jupiter.params.provider.EnumSource;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ThreadLocalRandom;
@@ -420,6 +421,180 @@ public class EqualiserCodeGeneratorTest {
.isEqualTo(equal);
}
+ @Test
+ public void testBinaryKeyMapEqualiser() {
+ DataType mapType = DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT());
+ RecordEqualiser equaliser =
+ new EqualiserCodeGenerator(new DataType[] {mapType})
+ .generateRecordEqualiser("binaryKeyMapFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+
+ // byte[] keys with equal content but distinct identity must compare
equal
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(singletonMap("k1".getBytes(), 1)),
+ new GenericMap(singletonMap("k1".getBytes(), 1)),
+ true);
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(singletonMap("k1".getBytes(), 1)),
+ new GenericMap(singletonMap("k2".getBytes(), 1)),
+ false);
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(singletonMap("k1".getBytes(), 1)),
+ new GenericMap(singletonMap("k1".getBytes(), 2)),
+ false);
+
+ // multiple keys: same insertion order on both sides so that the
serialized
+ // BinaryRow bytes are identical for the BinaryRow/BinaryRow fast path
+ Map<Object, Object> leftMap = new LinkedHashMap<>();
+ leftMap.put("k1".getBytes(), 1);
+ leftMap.put("k2".getBytes(), 2);
+ Map<Object, Object> rightMap = new LinkedHashMap<>();
+ rightMap.put("k1".getBytes(), 1);
+ rightMap.put("k2".getBytes(), 2);
+ assertMapEqualiser(
+ equaliser, mapType, new GenericMap(leftMap), new
GenericMap(rightMap), true);
+
+ // entry order must not matter for the element-wise comparison
+ Map<Object, Object> reversedMap = new LinkedHashMap<>();
+ reversedMap.put("k2".getBytes(), 2);
+ reversedMap.put("k1".getBytes(), 1);
+ assertThat(
+ equaliser.equals(
+ GenericRow.of(new GenericMap(leftMap)),
+ GenericRow.of(new GenericMap(reversedMap))))
+ .isTrue();
+
+ Map<Object, Object> differentValueMap = new LinkedHashMap<>();
+ differentValueMap.put("k1".getBytes(), 1);
+ differentValueMap.put("k2".getBytes(), 3);
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(leftMap),
+ new GenericMap(differentValueMap),
+ false);
+ }
+
+ @Test
+ public void testBinaryElementMultisetEqualiser() {
+ DataType multisetType = DataTypes.MULTISET(DataTypes.BYTES());
+ RecordEqualiser equaliser =
+ new EqualiserCodeGenerator(new DataType[] {multisetType})
+ .generateRecordEqualiser("binaryMultisetFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+
+ assertMapEqualiser(
+ equaliser,
+ multisetType,
+ new GenericMap(singletonMap("e1".getBytes(), 2)),
+ new GenericMap(singletonMap("e1".getBytes(), 2)),
+ true);
+ assertMapEqualiser(
+ equaliser,
+ multisetType,
+ new GenericMap(singletonMap("e1".getBytes(), 2)),
+ new GenericMap(singletonMap("e2".getBytes(), 2)),
+ false);
+ assertMapEqualiser(
+ equaliser,
+ multisetType,
+ new GenericMap(singletonMap("e1".getBytes(), 2)),
+ new GenericMap(singletonMap("e1".getBytes(), 3)),
+ false);
+ }
+
+ @Test
+ public void testRowKeyMapEqualiserAcrossRowRepresentations() {
+ DataType mapType = DataTypes.MAP(DataTypes.ROW(DataTypes.INT()),
DataTypes.INT());
+ RecordEqualiser equaliser =
+ new EqualiserCodeGenerator(new DataType[] {mapType})
+ .generateRecordEqualiser("rowKeyMapFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+
+ // a binary side yields NestedRow keys while the generic side yields
GenericRow keys
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(singletonMap(GenericRow.of(1), 10)),
+ new GenericMap(singletonMap(GenericRow.of(1), 10)),
+ true);
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(singletonMap(GenericRow.of(1), 10)),
+ new GenericMap(singletonMap(GenericRow.of(2), 10)),
+ false);
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(singletonMap(GenericRow.of(1), 10)),
+ new GenericMap(singletonMap(GenericRow.of(1), 11)),
+ false);
+ }
+
+ @Test
+ public void testIntKeyMapEqualiserIgnoresEntryOrder() {
+ DataType mapType = DataTypes.MAP(DataTypes.INT(), DataTypes.INT());
+ RecordEqualiser equaliser =
+ new EqualiserCodeGenerator(new DataType[] {mapType})
+ .generateRecordEqualiser("intKeyMapFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+
+ Map<Object, Object> leftMap = new LinkedHashMap<>();
+ leftMap.put(1, 10);
+ leftMap.put(2, 20);
+ Map<Object, Object> reversedMap = new LinkedHashMap<>();
+ reversedMap.put(2, 20);
+ reversedMap.put(1, 10);
+ assertThat(
+ equaliser.equals(
+ GenericRow.of(new GenericMap(leftMap)),
+ GenericRow.of(new GenericMap(reversedMap))))
+ .isTrue();
+ assertMapEqualiser(
+ equaliser, mapType, new GenericMap(leftMap), new
GenericMap(leftMap), true);
+
+ Map<Object, Object> differentValueMap = new LinkedHashMap<>();
+ differentValueMap.put(1, 10);
+ differentValueMap.put(2, 21);
+ assertMapEqualiser(
+ equaliser,
+ mapType,
+ new GenericMap(leftMap),
+ new GenericMap(differentValueMap),
+ false);
+ }
+
+ private static Map<Object, Object> singletonMap(Object key, Object value) {
+ Map<Object, Object> map = new HashMap<>();
+ map.put(key, value);
+ return map;
+ }
+
+ private static void assertMapEqualiser(
+ RecordEqualiser equaliser,
+ DataType mapType,
+ GenericMap left,
+ GenericMap right,
+ boolean equal) {
+ Serializer<?> serializer = InternalSerializers.create(mapType);
+ Function<GenericMap, BinaryRow> toBinaryRow =
+ value -> {
+ BinaryRow row = new BinaryRow(1);
+ BinaryRowWriter writer = new BinaryRowWriter(row);
+ BinaryWriter.write(writer, 0, value, mapType, serializer);
+ writer.complete();
+ return row;
+ };
+ assertBoolean(equaliser, toBinaryRow, left, right, equal);
+ }
+
@RepeatedTest(100)
public void testManyFields() {
int size = 499;
