This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 036b8895dc [spark] Convert EqualNullSafe(NaN) to isNaN in V2 filters
(#9720)
036b8895dc is described below
commit 036b8895dc499e2a9511711d3357f862b7ca1047
Author: cxzl25 <[email protected]>
AuthorDate: Fri Sep 11 13:43:14 2026 +0800
[spark] Convert EqualNullSafe(NaN) to isNaN in V2 filters (#9720)
---
.../paimon/spark/SparkV2FilterConverter.scala | 2 ++
.../spark/sql/SparkV2FilterConverterTestBase.scala | 25 ++++++++++++++++++++++
2 files changed, 27 insertions(+)
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
index 168c518cd3..41e5c47c4a 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
@@ -64,6 +64,8 @@ case class SparkV2FilterConverter(rowType: RowType) extends
Logging {
case BinaryPredicate(transform, literal) =>
if (literal == null) {
builder.isNull(transform)
+ } else if (isNaN(literal)) {
+ builder.isNaN(transform)
} else {
PredicateBuilder.and(builder.isNotNull(transform),
builder.equal(transform, literal))
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
index ea80fe476d..a865bce9ec 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
@@ -249,6 +249,31 @@ abstract class SparkV2FilterConverterTestBase extends
PaimonSparkTestBase {
sql("DROP TABLE IF EXISTS nan_tbl")
}
+ test("V2Filter: EqualNullSafe NaN") {
+ sql("CREATE TABLE nan_tbl (id INT, f FLOAT, d DOUBLE) USING paimon")
+ sql("INSERT INTO nan_tbl VALUES (1, 1.0, 1.0)")
+ sql("INSERT INTO nan_tbl VALUES (2, CAST('NaN' AS FLOAT), CAST('NaN' AS
DOUBLE))")
+ sql("INSERT INTO nan_tbl VALUES (3, 3.0, 3.0)")
+
+ val nanRowType = loadTable("nan_tbl").rowType()
+ val nanBuilder = new PredicateBuilder(nanRowType)
+ val nanConverter = SparkV2FilterConverter(nanRowType)
+
+ var filter = "f <=> CAST('NaN' AS FLOAT)"
+ var actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get
+ assert(actual.equals(nanBuilder.isNaN(1)))
+ checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(2)))
+ assert(scanFilesCount(filter, "nan_tbl") >= 1)
+
+ filter = "d <=> CAST('NaN' AS DOUBLE)"
+ actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get
+ assert(actual.equals(nanBuilder.isNaN(2)))
+ checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(2)))
+ assert(scanFilesCount(filter, "nan_tbl") >= 1)
+
+ sql("DROP TABLE IF EXISTS nan_tbl")
+ }
+
test("V2Filter: EqualNullSafe") {
var filter = "int_col <=> 1"
var actual = converter.convert(v2Filter(filter)).get
