This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7812b5f816 [common] Skip null literals in btree global index IN
predicates (#9624)
7812b5f816 is described below
commit 7812b5f816376bb5dbbbd87a33d21bd7c0d7bd04
Author: YangJie <[email protected]>
AuthorDate: Fri Sep 11 03:13:29 2026 -0400
[common] Skip null literals in btree global index IN predicates (#9624)
---
.../paimon/globalindex/btree/BTreeIndexReader.java | 5 ++++
.../globalindex/btree/AbstractIndexReaderTest.java | 24 +++++++++++++++++++
.../procedure/CreateGlobalIndexProcedureTest.scala | 28 ++++++++++++++++++++++
3 files changed, 57 insertions(+)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
index 6203d6d4a0..b0c272321c 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
@@ -334,6 +334,11 @@ public class BTreeIndexReader implements Closeable {
() -> {
RoaringNavigableMap64 result = new RoaringNavigableMap64();
for (Object literal : literals) {
+ // SQL IN treats NULL as never matching; skip it
instead of
+ // failing to serialize a null key.
+ if (literal == null) {
+ continue;
+ }
result.or(rangeQuery(literal, literal, true, true));
}
return result;
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java
index d35490e1fd..fe7ad8b40e 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java
@@ -276,6 +276,30 @@ public abstract class AbstractIndexReaderTest {
}
}
+ @TestTemplate
+ public void testInPredicateWithNullLiteral() throws Exception {
+ FieldRef ref = new FieldRef(1, "testField", dataType);
+
+ try (GlobalIndexReader reader = prepareDataAndCreateReader()) {
+ // Mixed null literal: SQL IN never matches NULL, so the null is
ignored
+ // and the remaining literals drive the result (previously NPE'd).
+ List<Object> literals =
+ new ArrayList<>(
+ data.get(0).getKey() == null
+ ? Collections.emptyList()
+ :
Collections.singletonList(data.get(0).getKey()));
+ literals.add(null);
+ GlobalIndexResult result = reader.visitIn(ref,
literals).join().get();
+ List<Object> finalLiterals = literals;
+ assertResult(result, filter(finalLiterals::contains));
+
+ // All-null list matches nothing.
+ GlobalIndexResult allNull =
+ reader.visitIn(ref,
Collections.singletonList(null)).join().get();
+ assertThat(allNull.results().isEmpty()).isTrue();
+ }
+ }
+
@TestTemplate
public void testStartsWith() throws Exception {
if (!dataType.is(DataTypeFamily.CHARACTER_STRING)) {
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala
index 90c1eda5cb..d051bc90dc 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala
@@ -448,6 +448,34 @@ class CreateGlobalIndexProcedureTest extends
PaimonSparkTestBase with StreamTest
}
}
+ test("btree global index IN predicate with more than 20 literals including
NULL") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, idx INT)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.enabled' = 'true',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true',
+ | 'btree-index.records-per-range' = '2')
+ |""".stripMargin)
+
+ spark.sql(s"INSERT INTO T VALUES ${(0 until 100).map(i => s"($i,
$i)").mkString(",")}")
+ createBTreeIndex("T", "idx")
+
+ // 21 distinct non-null literals plus NULL is 22 (> 20), so
PredicateBuilder keeps a real In
+ // leaf that reaches BTreeIndexReader.visitIn with a null (previously an
NPE). Pin
+ // inSetConversionThreshold high so the pushed predicate stays a plain
Catalyst In whose
+ // translation carries the null.
+ val inList = ((0 to 20).map(_.toString) :+ "NULL").mkString(", ")
+ withSQLConf("spark.sql.optimizer.inSetConversionThreshold" -> "100") {
+ checkAnswer(
+ sql(s"SELECT id FROM T WHERE idx IN ($inList) ORDER BY id"),
+ (0 to 20).map(Row(_)))
+ }
+ }
+ }
+
private def createBTreeIndex(tableName: String, column: String): Unit = {
spark
.sql(
