This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2b6a45e7e2c [fix](score) disable search topn with extra predicates
(#65821)
2b6a45e7e2c is described below
commit 2b6a45e7e2cb12d091ce0253366e50fca4efd5d3
Author: liangj777 <[email protected]>
AuthorDate: Fri Aug 21 14:40:18 2026 +0800
[fix](score) disable search topn with extra predicates (#65821)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
Search score TopN pushdown may return incorrect results when the search
predicate is combined with additional predicates, because the pushed
TopN limit can be applied before the remaining predicates are evaluated.
This change disables the pushed search TopN limit in those cases while
preserving the virtual score column pushdown, and adds regression
coverage for search predicates combined with equality, range, match,
score range and multiple search predicates.
---
.../rewrite/PushDownScoreTopNIntoOlapScan.java | 24 ++-
.../test_search_score_topn_predicates.out | 38 ++++
.../test_search_score_topn_predicates.groovy | 204 +++++++++++++++++++++
3 files changed, 263 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
index 68d206a4501..104f2b0f706 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java
@@ -194,17 +194,22 @@ public class PushDownScoreTopNIntoOlapScan implements
RewriteRuleFactory {
}
// When limit + offset overflows the long range, the pushed scan limit
would wrap to a
- // negative value; skip the push-down and let the TopN above the scan
apply limit/offset.
+ // negative value. Fail with the same error as ordinary TopN instead
of leaving score()
+ // unmaterialized and reporting an unrelated score() usage error.
if (Utils.addOverflows(topN.getLimit(), topN.getOffset())) {
- return null;
+ throw new AnalysisException("limit + offset overflows the long
range");
}
+ long scoreLimit = topN.getLimit() + topN.getOffset();
+ long pushedScoreLimit = shouldDisableSearchTopN(filter.getConjuncts(),
extractedScorePredicate)
+ ? 0L : scoreLimit;
+
// All conditions met, perform the push down.
// This is the core action: push score() as a virtual column and also
push the
// topN info.
Plan newScan =
scan.appendVirtualColumnsAndTopN(ImmutableList.of(scoreAlias),
ImmutableList.of(), Optional.empty(),
- topN.getOrderKeys(), Optional.of(topN.getLimit() +
topN.getOffset()),
+ topN.getOrderKeys(), Optional.of(pushedScoreLimit),
scoreRangeInfo);
// Rebuild the plan tree above the new scan.
@@ -243,6 +248,19 @@ public class PushDownScoreTopNIntoOlapScan implements
RewriteRuleFactory {
return topN.withChildren(newProject);
}
+ private boolean shouldDisableSearchTopN(Set<Expression> conjuncts,
Expression extractedScorePredicate) {
+ List<Expression> nonScoreConjuncts = conjuncts.stream()
+ .filter(conjunct -> extractedScorePredicate == null ||
!conjunct.equals(extractedScorePredicate))
+ .collect(ImmutableList.toImmutableList());
+
+ boolean hasSearchPredicate = nonScoreConjuncts.stream()
+ .anyMatch(conjunct -> !conjunct.collect(e -> e instanceof
SearchExpression).isEmpty());
+ if (!hasSearchPredicate) {
+ return false;
+ }
+ return nonScoreConjuncts.size() > 1 || !(nonScoreConjuncts.get(0)
instanceof SearchExpression);
+ }
+
/**
* Extract score range info from a single score predicate.
* Only supports min_score semantics (similar to Elasticsearch):
diff --git
a/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out
b/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out
new file mode 100644
index 00000000000..645855dd631
--- /dev/null
+++
b/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out
@@ -0,0 +1,38 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !single_search --
+1
+
+-- !search_with_equal_predicate --
+3
+
+-- !search_with_plain_equal_predicate --
+3
+
+-- !search_with_equal_limit_two --
+3
+4
+
+-- !search_with_equal_offset --
+4
+
+-- !search_with_match_predicate --
+3
+
+-- !search_with_range_predicate --
+3
+
+-- !multiple_search_predicates --
+4
+
+-- !search_with_score_range_only --
+1
+3
+
+-- !search_with_score_range_and_other_predicate --
+3
+
+-- !nested_search_with_other_predicate --
+7
+
+-- !not_search_with_other_search --
+6
diff --git
a/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy
b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy
new file mode 100644
index 00000000000..df780835751
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_topn_predicates", "p0") {
+ sql "DROP TABLE IF EXISTS test_search_score_topn_predicates"
+
+ sql """
+ CREATE TABLE test_search_score_topn_predicates (
+ id INT,
+ status VARCHAR(20),
+ plain_status VARCHAR(20),
+ category VARCHAR(20),
+ title TEXT,
+ body TEXT,
+ INDEX idx_status (status) USING INVERTED,
+ INDEX idx_category (category) USING INVERTED,
+ INDEX idx_title (title) USING INVERTED PROPERTIES("parser" =
"english", "support_phrase" = "true"),
+ INDEX idx_body (body) USING INVERTED PROPERTIES("parser" =
"english", "support_phrase" = "true")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true"
+ )
+ """
+
+ sql """
+ INSERT INTO test_search_score_topn_predicates VALUES
+ (1, 'drop', 'drop', 'plain', 'apple apple apple apple apple apple
apple apple apple apple apple apple', 'alpha'),
+ (2, 'keep', 'keep', 'plain', 'apple', 'alpha'),
+ (3, 'keep', 'keep', 'plain', 'apple apple apple apple apple',
'alpha'),
+ (4, 'keep', 'keep', 'plain', 'apple apple apple', 'beta beta
beta'),
+ (5, 'drop', 'drop', 'plain', 'banana', 'beta beta beta beta beta
beta beta beta beta beta beta beta'),
+ (6, 'keep', 'keep', 'plain', 'pear', 'beta beta beta beta beta'),
+ (7, 'keep', 'keep', 'special', 'cherry cherry cherry cherry',
'gamma'),
+ (8, 'drop', 'drop', 'special', 'cherry cherry cherry cherry cherry
cherry cherry cherry cherry cherry', 'gamma')
+ """
+
+ sql "sync"
+ sql "set enable_nereids_planner = true"
+ sql "set enable_fallback_to_original_planner = false"
+ sql "set enable_segment_limit_pushdown = true"
+ sql "set enable_inverted_index_query_cache = false"
+
+ qt_single_search """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_equal_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_plain_equal_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND plain_status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_equal_limit_two """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 2
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_equal_offset """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1 OFFSET 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_match_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND status MATCH 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_range_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND id > 1
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_multiple_search_predicates """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND search('body:beta')
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_score_range_only """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND score() > 0
+ ORDER BY s DESC
+ LIMIT 2
+ ) t
+ ORDER BY id
+ """
+
+ qt_search_with_score_range_and_other_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple') AND score() > 0 AND status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_nested_search_with_other_predicate """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE (search('title:cherry') OR category MATCH 'special') AND
status = 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ qt_not_search_with_other_search """
+ SELECT id FROM (
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE NOT search('title:apple') AND search('body:beta') AND status
= 'keep'
+ ORDER BY s DESC
+ LIMIT 1
+ ) t
+ ORDER BY id
+ """
+
+ // limit + offset overflows the long range. score() must report the
standard TopN
+ // overflow error instead of skipping score pushdown and reporting a
score() usage error.
+ test {
+ sql """
+ SELECT id, score() AS s
+ FROM test_search_score_topn_predicates
+ WHERE search('title:apple')
+ ORDER BY s DESC
+ LIMIT 9223372036854775807 OFFSET 9223372036854775807
+ """
+ exception "limit + offset overflows the long range"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
