This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1b6f6cbe9a [core][spark][python] Reject non-finite hybrid route
weights (#8327)
1b6f6cbe9a is described below
commit 1b6f6cbe9acccd591dc0ff4a471378d3f48933a1
Author: QuakeWang <[email protected]>
AuthorDate: Tue Jun 23 19:26:48 2026 +0800
[core][spark][python] Reject non-finite hybrid route weights (#8327)
Hybrid search route weights were only checked with `weight <= 0`. `NaN`
bypassed that check, while `Infinity` was accepted and could dominate
RRF or weighted-score fusion.
This PR requires route weights to be finite and positive across Java,
Spark TVF parsing, and Python. It also validates direct
`HybridSearchRanker` weight inputs through `WeightedResult`.
---
.../paimon/globalindex/HybridSearchRanker.java | 10 ++++-
.../apache/paimon/predicate/HybridSearchRoute.java | 5 ++-
.../paimon/globalindex/HybridSearchRankerTest.java | 31 ++++++++++++++
.../apache/paimon/predicate/FullTextQueryTest.java | 37 +++++++++++++++++
.../pypaimon/table/source/hybrid_search_builder.py | 6 ++-
.../pypaimon/tests/hybrid_search_ranker_test.py | 16 ++++++++
.../plans/logical/PaimonTableValuedFunctions.scala | 7 +++-
.../plans/logical/VectorSearchQueryTest.scala | 47 ++++++++++++++++++++++
8 files changed, 152 insertions(+), 7 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java
index eb2f1d45ac..1b6504c0f9 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/HybridSearchRanker.java
@@ -182,6 +182,14 @@ public class HybridSearchRanker {
return weights[index];
}
+ private static float checkWeight(float weight) {
+ if (!Float.isFinite(weight) || weight <= 0) {
+ throw new IllegalArgumentException(
+ "Weight must be finite and positive, got: " + weight);
+ }
+ return weight;
+ }
+
/** Weighted result from one search route. */
public static class WeightedResult implements Serializable {
@@ -192,7 +200,7 @@ public class HybridSearchRanker {
public WeightedResult(ScoredGlobalIndexResult result, float weight) {
this.result = result;
- this.weight = weight;
+ this.weight = checkWeight(weight);
}
public ScoredGlobalIndexResult result() {
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/HybridSearchRoute.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/HybridSearchRoute.java
index 57e5369bd4..b0746635bf 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/HybridSearchRoute.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/HybridSearchRoute.java
@@ -104,8 +104,9 @@ public class HybridSearchRoute implements Serializable {
if (limit <= 0) {
throw new IllegalArgumentException("Limit must be positive, got: "
+ limit);
}
- if (weight <= 0) {
- throw new IllegalArgumentException("Weight must be positive, got:
" + weight);
+ if (!Float.isFinite(weight) || weight <= 0) {
+ throw new IllegalArgumentException(
+ "Weight must be finite and positive, got: " + weight);
}
this.routeType = routeType;
this.fieldName = fieldName;
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/HybridSearchRankerTest.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/HybridSearchRankerTest.java
index f927d0cf8a..0d4b2c0824 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/HybridSearchRankerTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/HybridSearchRankerTest.java
@@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.within;
/** Tests for {@link HybridSearchRanker}. */
@@ -126,6 +127,36 @@ public class HybridSearchRankerTest {
assertThat(ranked.scoreGetter().score(3L)).isCloseTo(2.0f,
within(0.000001f));
}
+ @Test
+ public void testRejectNonFiniteWeights() {
+ ScoredGlobalIndexResult result = result(new long[] {1}, new float[]
{1.0f});
+
+ assertThatThrownBy(
+ () ->
+ HybridSearchRanker.rrf(
+ Collections.singletonList(result),
+ new float[] {Float.NaN},
+ 1))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Weight must be finite and positive");
+
+ assertThatThrownBy(
+ () ->
+ HybridSearchRanker.weightedScore(
+ Collections.singletonList(result),
+ new float[] {Float.POSITIVE_INFINITY},
+ 1))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Weight must be finite and positive");
+
+ assertThatThrownBy(
+ () ->
+ new HybridSearchRanker.WeightedResult(
+ result, Float.NEGATIVE_INFINITY))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Weight must be finite and positive");
+ }
+
private ScoredGlobalIndexResult result(long[] rowIds, float[] scores) {
RoaringNavigableMap64 bitmap = new RoaringNavigableMap64();
return result(rowIds, scores, bitmap);
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/FullTextQueryTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/FullTextQueryTest.java
index 5fc34ecd34..ff8ad91b76 100644
---
a/paimon-common/src/test/java/org/apache/paimon/predicate/FullTextQueryTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/FullTextQueryTest.java
@@ -199,4 +199,41 @@ public class FullTextQueryTest {
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Full-text hybrid route options are not
supported yet");
}
+
+ @Test
+ public void testHybridRouteRejectsNonFiniteWeight() {
+ assertThatThrownBy(
+ () ->
+ HybridSearchRoute.vector(
+ "embedding",
+ new float[] {1.0f},
+ 10,
+ Float.NaN,
+ Collections.emptyMap()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Weight must be finite and positive");
+
+ assertThatThrownBy(
+ () ->
+ HybridSearchRoute.vector(
+ "embedding",
+ new float[] {1.0f},
+ 10,
+ Float.POSITIVE_INFINITY,
+ Collections.emptyMap()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Weight must be finite and positive");
+
+ assertThatThrownBy(
+ () ->
+ HybridSearchRoute.builder()
+ .query(
+
"{\"match\":{\"column\":\"content\","
+ + "\"terms\":\"paimon
lake\"}}")
+ .limit(10)
+ .weight(Float.NEGATIVE_INFINITY)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Weight must be finite and positive");
+ }
}
diff --git a/paimon-python/pypaimon/table/source/hybrid_search_builder.py
b/paimon-python/pypaimon/table/source/hybrid_search_builder.py
index 3415d3bf9f..8edeeac251 100644
--- a/paimon-python/pypaimon/table/source/hybrid_search_builder.py
+++ b/paimon-python/pypaimon/table/source/hybrid_search_builder.py
@@ -18,6 +18,7 @@
"""Builder to build hybrid search."""
import heapq
+import math
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Dict, List, Optional
@@ -77,8 +78,9 @@ class HybridSearchRoute:
"Query cannot be None for full-text route")
if self.limit <= 0:
raise ValueError("Limit must be positive, got: %s" % self.limit)
- if self.weight <= 0:
- raise ValueError("Weight must be positive, got: %s" % self.weight)
+ if not math.isfinite(self.weight) or self.weight <= 0:
+ raise ValueError(
+ "Weight must be finite and positive, got: %s" % self.weight)
self.options = dict(self.options or {})
if self.route_type == self.FULL_TEXT:
_check_full_text_options(self.options)
diff --git a/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
b/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
index aa4936dec2..47dc85a428 100644
--- a/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
+++ b/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
@@ -19,6 +19,7 @@
HybridSearchRankerTest so the weighted_score ranker stays consistent across
languages (per-route min-max normalization before weighting)."""
+import math
import unittest
from pypaimon.globalindex.vector_search_result import
DictBasedScoredIndexResult
@@ -81,6 +82,21 @@ class HybridSearchRankerTest(unittest.TestCase):
# Rank-based fusion respects the 5x vector weight: rowId 1 wins.
self.assertGreater(getter(1), getter(2))
+ def test_route_rejects_non_finite_weight(self):
+ for weight in (math.nan, math.inf, -math.inf):
+ with self.subTest(weight=weight):
+ with self.assertRaisesRegex(
+ ValueError, "Weight must be finite and positive"):
+ HybridSearchRoute.vector_route(
+ "f", [1.0], 10, weight=weight)
+
+ with self.assertRaisesRegex(
+ ValueError, "Weight must be finite and positive"):
+ HybridSearchRoute.full_text_route(
+ '{"match":{"column":"content","terms":"paimon lake"}}',
+ 10,
+ weight=math.inf)
+
if __name__ == "__main__":
unittest.main()
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
index bf3e53bf2f..7d41d7f1b5 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
@@ -626,8 +626,11 @@ case class HybridSearchQuery(override val args:
Seq[Expression])
case u: UTF8String => u.toString.toFloat
case other => throw new RuntimeException(s"Invalid $name type:
${other.getClass.getName}")
}
- if (parsed <= 0) {
- throw new IllegalArgumentException(s"$name must be positive, but got:
$parsed")
+ if (!java.lang.Float.isFinite(parsed) || parsed <= 0) {
+ if (name == "weight") {
+ throw new IllegalArgumentException(s"Weight must be finite and
positive, got: $parsed")
+ }
+ throw new IllegalArgumentException(s"$name must be finite and positive,
but got: $parsed")
}
parsed
}
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/catalyst/plans/logical/VectorSearchQueryTest.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/catalyst/plans/logical/VectorSearchQueryTest.scala
index 82a91fc9ee..08bc308182 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/catalyst/plans/logical/VectorSearchQueryTest.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/catalyst/plans/logical/VectorSearchQueryTest.scala
@@ -156,6 +156,53 @@ class VectorSearchQueryTest extends AnyFunSuite {
assert(exception.getMessage.contains("Full-text hybrid route options are
not supported yet"))
}
+ test("reject hybrid route with non-finite weight") {
+ val vectorException = intercept[IllegalArgumentException] {
+ HybridSearchQuery(Seq.empty).createHybridSearch(
+ innerTable,
+ Seq(
+ CreateArray(
+ Seq(
+ CreateNamedStruct(Seq(
+ Literal("vector_column"),
+ Literal("title_vec"),
+ Literal("query_vector"),
+ CreateArray(Seq(Literal(1.0f), Literal(0.0f))),
+ Literal("weight"),
+ Literal(Float.NaN)
+ ))
+ )),
+ CreateArray(Seq.empty),
+ Literal(5)
+ )
+ )
+ }
+
+ assert(vectorException.getMessage.contains("Weight must be finite and
positive"))
+
+ val fullTextException = intercept[IllegalArgumentException] {
+ HybridSearchQuery(Seq.empty).createHybridSearch(
+ innerTable,
+ Seq(
+ CreateArray(Seq.empty),
+ CreateArray(
+ Seq(
+ CreateNamedStruct(
+ Seq(
+ Literal("query"),
+ Literal("""{"match":{"column":"content","terms":"paimon
lake"}}"""),
+ Literal("weight"),
+ Literal(Float.PositiveInfinity)
+ ))
+ )),
+ Literal(5)
+ )
+ )
+ }
+
+ assert(fullTextException.getMessage.contains("Weight must be finite and
positive"))
+ }
+
test("create full-text search") {
val search = FullTextSearchQuery(Seq.empty).createFullTextSearch(
innerTable,
