This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4016f207bfe [fix](be) Support BM25 scoring on keyword SEARCH indexes
(#69051)
4016f207bfe is described below
commit 4016f207bfe7eff77f56f4d98d23629888539462
Author: Jack <[email protected]>
AuthorDate: Tue Oct 20 10:29:45 2026 +0800
[fix](be) Support BM25 scoring on keyword SEARCH indexes (#69051)
### What problem does this PR solve?
Issue Number: None
Related PR: #68758
Problem Summary: SEARCH can match a keyword inverted index, but
projecting score() can drop matched rows or report an unsupported index.
Keyword indexes omit norms and may omit frequencies; they still need
usable BM25 defaults.
Preserve whole keyword values, including case and whitespace. Use
frequency and encoded norm 1 for docs-only/no-norm postings. Legacy
indexes without total-term-frequency metadata use average length 1; SNII
keeps its physical statistics. Analyzed segments missing required
scoring metadata and numeric BKD scoring retain their existing
restrictions.
This branch incorporates master's unified MATCH/SEARCH engine from
#68758. Keyword norm handling lives in the CLucene postings cursor,
including partition switches and WAND bounds; SNII cursors provide
docs-only frequencies and norms. The obsolete query_v2/SNII scoring
implementations are removed with master.
Keyword matching and missing-metadata behavior follow Elasticsearch
while retaining Doris's existing BM25 formula and collection-statistics
conventions.
#### P1 follow-up: keyword filters and internal conjunctions
1. A keyword index explicitly declared with `parser=none` can inherit
`support_phrase=true` from FE defaults. Ordinary MATCH filters then
entered scoring although the statistics collector skipped that field.
Both IndexReaderHelper query-type overloads now share the canonical
analyzer-and-phrase eligibility check used by statistics collection. The
implementation is out of line to keep analyzer dependencies out of the
header. Named-analyzer and normalizer behavior is preserved. Equality
and keyword MATCH filters keep the SEARCH-only baseline score.
2. Scored SNII AND grouped a docs-only keyword term into a path that
counts positions. Such sources now use the existing individual term
scorer with implicit unit frequency; position-bearing sources retain
batched scoring. Internal `SEARCH('body:apple AND tag:x')` continues to
score both fields.
---
be/src/storage/index/index_reader_helper.cpp | 54 +++++
be/src/storage/index/index_reader_helper.h | 30 +--
.../inverted/clucene/spi/clucene_postings_cursor.h | 6 +-
.../engine/boolean_query/operator_boolean_weight.h | 9 +
.../index/inverted/query/scoring/bm25_similarity.h | 10 +-
.../inverted/reader/collection_statistics.cpp | 37 ++--
.../index/inverted/reader/collection_statistics.h | 10 +-
.../query/engine/boolean_postings_bench_test.cpp | 2 +
.../engine/boolean_query/listed_terms_test.cpp | 26 +++
.../query/engine/multi_segment_collector_test.cpp | 2 +
.../query/engine/segment_postings_test.cpp | 42 ++++
.../index/inverted/query/spi/fake_index_source.h | 34 ++-
.../inverted/reader/collection_statistics_test.cpp | 59 +++--
.../storage/segment/index_reader_helper_test.cpp | 51 +++++
.../search/test_search_score_keyword_filters.out | 240 +++++++++++++++++++++
.../data/search/test_search_score_raw_index.out | 205 ++++++++++++++++++
.../test_search_score_keyword_filters.groovy | 85 ++++++++
.../search/test_search_score_raw_index.groovy | 112 ++++++++++
18 files changed, 930 insertions(+), 84 deletions(-)
diff --git a/be/src/storage/index/index_reader_helper.cpp
b/be/src/storage/index/index_reader_helper.cpp
new file mode 100644
index 00000000000..5607169cd08
--- /dev/null
+++ b/be/src/storage/index/index_reader_helper.cpp
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/index_reader_helper.h"
+
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+
+namespace doris::segment_v2 {
+namespace {
+
+bool can_score_match(const TabletIndex* index_meta) {
+ const auto& properties = index_meta->properties();
+ // parser=none can inherit support_phrase=true from FE defaults. MATCH
statistics skip
+ // these keyword indexes, so their filters must not request a scoring
context either.
+ return inverted_index::InvertedIndexAnalyzer::should_analyzer(properties)
&&
+ (get_parser_phrase_support_string_from_properties(properties) ==
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ IndexReaderHelper::is_normalizer_only(properties));
+}
+
+} // namespace
+
+bool IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType
query_type,
+ const TabletIndex*
index_meta) {
+ return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) &&
+ can_score_match(index_meta);
+}
+
+bool IndexReaderHelper::is_need_similarity_score(TExprOpcode::type query_type,
+ const TabletIndex*
index_meta) {
+ return (query_type == TExprOpcode::MATCH_ANY || query_type ==
TExprOpcode::MATCH_ALL ||
+ query_type == TExprOpcode::MATCH_PHRASE ||
+ query_type == TExprOpcode::MATCH_PHRASE_PREFIX) &&
+ can_score_match(index_meta);
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/storage/index/index_reader_helper.h
b/be/src/storage/index/index_reader_helper.h
index c1b0b65008d..ffdb0a30075 100644
--- a/be/src/storage/index/index_reader_helper.h
+++ b/be/src/storage/index/index_reader_helper.h
@@ -103,36 +103,12 @@ public:
return iter->get_reader(InvertedIndexReaderType::STRING_TYPE) !=
nullptr;
}
+ // MATCH filters score analyzed phrase indexes and single-term normalizers.
static bool is_need_similarity_score(InvertedIndexQueryType query_type,
- const TabletIndex* index_meta) {
- if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
- const auto& properties = index_meta->properties();
- if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
- is_normalizer_only(properties)) {
- return true;
- }
- }
- return false;
- }
+ const TabletIndex* index_meta);
static bool is_need_similarity_score(TExprOpcode::type query_type,
- const TabletIndex* index_meta) {
- if (query_type == TExprOpcode::MATCH_ANY || query_type ==
TExprOpcode::MATCH_ALL ||
- query_type == TExprOpcode::MATCH_PHRASE ||
- query_type == TExprOpcode::MATCH_PHRASE_PREFIX) {
- const auto& properties = index_meta->properties();
- if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
- is_normalizer_only(properties)) {
- return true;
- }
- }
- return false;
- }
+ const TabletIndex* index_meta);
};
} // namespace doris::segment_v2
diff --git
a/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
b/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
index 2bb3406eaad..9bc5b9328c1 100644
--- a/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
+++ b/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
@@ -155,7 +155,7 @@ public:
const int32_t last = _iter->getLastDocInBlock();
_bound = {.last_doc = static_cast<uint32_t>(last),
.max_freq = _iter->getMaxBlockFreq(),
- .max_norm = _iter->getMaxBlockNorm(),
+ .max_norm = _has_norms ? _iter->getMaxBlockNorm() : 1,
.last_doc_known = last >= 0 && last != INT_MAX};
if (_bound->last_doc_known) {
_bound->last_doc += _doc_base;
@@ -305,6 +305,8 @@ public:
}
private:
+ // Keyword indexes omit norms even when CLucene supplies a zero-filled
norm buffer.
+ // Exposing no norms makes PostingsBlock use length one, including after
partition switches.
void _select_partition(size_t index) {
_partition = index;
_block = {};
@@ -314,8 +316,8 @@ private:
const auto& part = _partitions[index];
_iter = part.iter.get();
_raw_positions = part.positions;
- _doc_base = part.begin;
_has_norms = part.has_norms;
+ _doc_base = part.begin;
}
}
diff --git
a/be/src/storage/index/inverted/query/engine/boolean_query/operator_boolean_weight.h
b/be/src/storage/index/inverted/query/engine/boolean_query/operator_boolean_weight.h
index cd092de9405..fd4e4c5905b 100644
---
a/be/src/storage/index/inverted/query/engine/boolean_query/operator_boolean_weight.h
+++
b/be/src/storage/index/inverted/query/engine/boolean_query/operator_boolean_weight.h
@@ -209,6 +209,15 @@ private:
if (source == nullptr || !(any_source || source->batches_reads()))
{
continue;
}
+ if (scoring && _type == OperatorType::OP_AND) {
+ index_query::IndexScoringStats stats;
+ THROW_IF_ERROR(source->scoring_stats(&stats));
+ // Batched AND scoring counts positions. Docs-only keywords
have none, so
+ // keep their individual scorers, which supply implicit unit
frequencies.
+ if (!stats.has_positions) {
+ continue;
+ }
+ }
auto group = std::ranges::find_if(groups, [&source](const
ListedTerms& candidate) {
return candidate.source() == source;
});
diff --git a/be/src/storage/index/inverted/query/scoring/bm25_similarity.h
b/be/src/storage/index/inverted/query/scoring/bm25_similarity.h
index 535a2271efc..73e74422edb 100644
--- a/be/src/storage/index/inverted/query/scoring/bm25_similarity.h
+++ b/be/src/storage/index/inverted/query/scoring/bm25_similarity.h
@@ -24,14 +24,8 @@
namespace doris::segment_v2 {
-/**
- * BM25 similarity scoring implementation for inverted index queries.
- *
- * Note: Currently, BM25 scoring is only applied to tokenized indexes (e.g.,
standard, chinese analyzers)
- * and not to non-tokenized indexes (e.g., keyword analyzer). This differs
from Elasticsearch,
- * which applies scoring to keyword indexes as well. In Doris,
keyword/non-tokenized indexes
- * are primarily used for exact matching without relevance scoring.
- */
+// BM25 scoring for analyzed and keyword indexes. Keyword indexes without norms
+// use the Lucene default encoded norm of one.
class BM25Similarity : public Similarity {
public:
BM25Similarity();
diff --git a/be/src/storage/index/inverted/reader/collection_statistics.cpp
b/be/src/storage/index/inverted/reader/collection_statistics.cpp
index 276b1d2c2dd..07e3fa866d6 100644
--- a/be/src/storage/index/inverted/reader/collection_statistics.cpp
+++ b/be/src/storage/index/inverted/reader/collection_statistics.cpp
@@ -44,9 +44,11 @@ namespace collection_statistics_detail {
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t
index_doc_count,
uint64_t
sum_total_term_freq,
bool
has_positions, bool has_norms,
+ bool is_analyzed,
bool
single_term_normalizer) {
- // Lowercase-normalized docs-only postings use unit frequency and length
for scoring.
- if ((!has_positions || !has_norms) && !single_term_normalizer) {
+ // Keywords and lowercase-only normalizers can omit both sections: a
matched value
+ // contributes one term with unit frequency and length. Other analyzed
sources need both.
+ if (is_analyzed && !single_term_normalizer && (!has_positions ||
!has_norms)) {
return
ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, false>(
"SNII scoring requires positions and norms; this segment was
written without "
"positions or without norms. Norms are left out when the index
sets \"norms\" = "
@@ -264,10 +266,11 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
const uint64_t segment_doc_count = scoring_stats.doc_count;
RETURN_IF_ERROR(admit_snii_scoring_segment(
ws_field_name, segment_doc_count,
scoring_stats.token_count,
- scoring_stats.has_positions, scoring_stats.has_norms,
- segment_v2::IndexReaderHelper::is_normalizer_only(
+ scoring_stats.has_positions, scoring_stats.has_norms,
&segment_accumulator,
+
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
collect_info.index_meta->properties()),
- &segment_accumulator));
+ segment_v2::IndexReaderHelper::is_normalizer_only(
+ collect_info.index_meta->properties())));
for (const auto& logical_term_bytes : collect_info.unique_terms) {
const auto logical_term =
@@ -337,12 +340,11 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
// BM25 on an analyzed index needs the record length of every row, and
CLucene keeps
// them, together with the field's token count, in the norms. A
segment written without
// norms would feed a zero avgdl, or rank its rows as zero-length
documents next to the
- // segments that have norms, so refuse to score the collection, as
SNII does. A
- // single-term normalizer can instead use unit frequency and length.
- const bool single_term_normalizer =
segment_v2::IndexReaderHelper::is_normalizer_only(
- collect_info.index_meta->properties());
- // Docs-only lowercase indexes have no norms, but each matching row
has one term.
- if (!scoring_stats.has_norms && !single_term_normalizer &&
+ // segments that have norms, so refuse to score the collection, as
SNII does.
+ // Keywords and lowercase-only normalizers instead use unit document
length.
+ if (!scoring_stats.has_norms &&
+ !segment_v2::IndexReaderHelper::is_normalizer_only(
+ collect_info.index_meta->properties()) &&
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
collect_info.index_meta->properties())) {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
@@ -351,10 +353,8 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
"for a variant path, when
inverted_index_skip_norms_for_variant is on",
seg_path,
segment_v2::inverted_index::StringHelper::to_string(ws_field_name));
}
- // CLucene omits term totals without norms; one token per row gives a
unit-length estimate.
- _total_num_tokens[ws_field_name] += !scoring_stats.has_norms &&
single_term_normalizer
- ? scoring_stats.doc_count
- :
scoring_stats.token_count;
+ _total_num_tokens[ws_field_name] +=
+ scoring_stats.has_norms ? scoring_stats.token_count :
scoring_stats.doc_count;
for (const auto& logical_term_bytes : collect_info.unique_terms) {
const auto logical_term =
@@ -375,11 +375,12 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
Status CollectionStatistics::admit_snii_scoring_segment(
const std::wstring& field_name, uint64_t index_doc_count, uint64_t
sum_total_term_freq,
- bool has_positions, bool has_norms, bool single_term_normalizer,
- SniiScoringSegmentAccumulator* segment_accumulator) {
+ bool has_positions, bool has_norms, SniiScoringSegmentAccumulator*
segment_accumulator,
+ bool is_analyzed, bool single_term_normalizer) {
DORIS_CHECK(segment_accumulator != nullptr);
auto segment_stats =
collection_statistics_detail::resolve_snii_scoring_segment(
- index_doc_count, sum_total_term_freq, has_positions, has_norms,
single_term_normalizer);
+ index_doc_count, sum_total_term_freq, has_positions, has_norms,
is_analyzed,
+ single_term_normalizer);
if (!segment_stats.has_value()) {
clear();
return segment_stats.error();
diff --git a/be/src/storage/index/inverted/reader/collection_statistics.h
b/be/src/storage/index/inverted/reader/collection_statistics.h
index e62258c2b81..27b61b6ad75 100644
--- a/be/src/storage/index/inverted/reader/collection_statistics.h
+++ b/be/src/storage/index/inverted/reader/collection_statistics.h
@@ -86,8 +86,9 @@ private:
io::IOContext* io_ctx);
Status admit_snii_scoring_segment(const std::wstring& field_name, uint64_t
index_doc_count,
uint64_t sum_total_term_freq, bool
has_positions,
- bool has_norms, bool
single_term_normalizer,
- SniiScoringSegmentAccumulator*
segment_accumulator);
+ bool has_norms,
+ SniiScoringSegmentAccumulator*
segment_accumulator,
+ bool is_analyzed = true, bool
single_term_normalizer = false);
void commit_snii_scoring_segment(SniiScoringSegmentAccumulator&&
segment_accumulator);
void clear();
@@ -121,11 +122,12 @@ struct SniiScoringSegmentStats {
uint64_t token_count = 0;
};
-// Single-term normalizers can score docs-only postings with frequency and
length one.
-// Other analyzed indexes need positions and norms for accurate BM25 scores.
+// Analyzed SNII indexes need positions and norms. Keywords and lowercase-only
+// normalizers can use unit frequency and norm when those sections are absent.
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t
index_doc_count,
uint64_t
sum_total_term_freq,
bool
has_positions, bool has_norms,
+ bool is_analyzed
= true,
bool
single_term_normalizer = false);
void add_term_doc_frequency(
diff --git
a/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
b/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
index 6e1e2dfc717..b99f525aca4 100644
---
a/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
+++
b/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
@@ -202,6 +202,7 @@ uint64_t posting_row_checksum(uint32_t doc, float score,
bool scoring) {
uint64_t expected_posting_checksum(const PostingSpec& spec, uint32_t profile,
bool sparse,
bool scoring) {
BM25Similarity similarity(2.0F, 8.0F);
+ // The physical fixtures omit norms, so every document has unit length.
const float term_score = similarity.score(1.0F, 1);
uint64_t result = 0;
for (uint32_t doc = 0; doc < kPostingRows; ++doc) {
@@ -255,6 +256,7 @@ PhysicalTruthRows expected_physical_truth(const
PostingSpec& spec, uint32_t prof
PhysicalTruthRows result;
result.scores.resize(kPostingRows);
BM25Similarity similarity(2.0F, 8.0F);
+ // The physical fixtures omit norms, so every document has unit length.
const float term_score = similarity.score(1.0F, 1);
for (uint32_t doc = 0; doc < kPostingRows; ++doc) {
bool required_true = true;
diff --git
a/be/test/storage/index/inverted/query/engine/boolean_query/listed_terms_test.cpp
b/be/test/storage/index/inverted/query/engine/boolean_query/listed_terms_test.cpp
index fe3ec36a32f..c3ab4f94e41 100644
---
a/be/test/storage/index/inverted/query/engine/boolean_query/listed_terms_test.cpp
+++
b/be/test/storage/index/inverted/query/engine/boolean_query/listed_terms_test.cpp
@@ -687,6 +687,32 @@ TEST_F(ListedTermsTest,
AScoredConjunctionScoresTheRowsItLists) {
EXPECT_TRUE(listed->opened.empty());
}
+// A batching keyword source has no positions: its term scorers use frequency
and norm 1.
+TEST_F(ListedTermsTest,
AScoredDocsOnlyConjunctionUsesUnitFrequencyWithoutPositions) {
+ for (bool batches : {false, true}) {
+ auto keyword = std::make_shared<FakeIndexSource>();
+ keyword->batches = batches;
+ keyword->has_positions = false;
+ keyword->set_doc_count(10);
+ keyword->add("a", std::vector<uint32_t> {1, 2, 3});
+ keyword->add("b", std::vector<uint32_t> {2, 3, 4});
+ auto expected = summed({"a", "b"}, keyword);
+ expected.erase(1);
+ expected.erase(4);
+ keyword->opened.clear();
+ const auto actual = scored(
+ [&] {
+ return boolean(OperatorType::OP_AND, {term("a"),
term("b")});
+ },
+ keyword);
+ expect_scores_eq(actual, expected);
+ EXPECT_EQ(keys(actual), (std::vector<uint32_t> {2, 3}));
+ EXPECT_GT(actual.at(2), 0.0F);
+ EXPECT_EQ(keyword->opened, (std::vector<std::string> {"a", "b"}));
+ EXPECT_TRUE(keyword->opened_together.empty());
+ }
+}
+
// The chain lists the rarer term whole and the other on its rows; then both
read the positions
// of the rows holding every term, in one round.
TEST_F(ListedTermsTest,
AScoredConjunctionReadsTheListedRowsPositionsInOneRound) {
diff --git
a/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
b/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
index d2e096fffc0..74affbaa8be 100644
---
a/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
+++
b/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
@@ -249,6 +249,7 @@ static void check_native_posting_block(const
index_query::PostingsBlock& block,
EXPECT_EQ(block.doc_at(0), doc);
EXPECT_EQ(block.docs.front(), doc);
EXPECT_EQ(block.freq_at(0), 1);
+ // These segments omit norms; crossing a partition must retain unit length.
EXPECT_TRUE(block.norms.empty());
EXPECT_EQ(block.norm_at(0), 1);
}
@@ -408,6 +409,7 @@ TEST_F(MultiSegmentCollectorTest,
PhraseCandidatesRemainValidAfterWeightDestruct
auto scorer = weight->scorer(segment, "");
weight.reset();
ASSERT_EQ(scorer->doc(), 3);
+ // Materialized phrase postings retain the no-norm source's unit
length.
const float expected_score = scoring ? similarity->score(1.0F, 1)
: 1.0F;
EXPECT_FLOAT_EQ(scorer->score(), expected_score);
EXPECT_EQ(scorer->seek(3), 3);
diff --git
a/be/test/storage/index/inverted/query/engine/segment_postings_test.cpp
b/be/test/storage/index/inverted/query/engine/segment_postings_test.cpp
index 402965dec88..06951645cef 100644
--- a/be/test/storage/index/inverted/query/engine/segment_postings_test.cpp
+++ b/be/test/storage/index/inverted/query/engine/segment_postings_test.cpp
@@ -1282,6 +1282,48 @@ TEST_F(SegmentPostingsTest,
test_segment_postings_base_constructor_next_true) {
EXPECT_EQ(base.norm(), 1);
}
+// A leaf cursor exposes block bounds; the general mock deliberately inherits
the
+// unsupported multi-reader bound methods and cannot exercise keyword WAND
defaults.
+class KeywordTermDocs final : public MockTermDocs {
+public:
+ using MockTermDocs::MockTermDocs;
+ int32_t getLastDocInBlock() override { return
static_cast<int32_t>(_docs.back()); }
+ int32_t getMaxBlockFreq() override { return -1; }
+ int32_t getMaxBlockNorm() override { return -1; }
+};
+
+TEST_F(SegmentPostingsTest, KeywordPostingsUseUnitNorm) {
+ ClucenePostingsCursor::Partition part {
+ .iter = TermDocsPtr(new KeywordTermDocs({1, 3}, {1, 1}, {0, 0},
2)),
+ .has_norms = false};
+ auto cursor = std::make_unique<ClucenePostingsCursor>(std::move(part));
+ auto similarity = std::make_shared<BM25Similarity>(2.0F, 1.0F);
+ SegmentPostings postings(std::move(cursor), true, similarity);
+ EXPECT_EQ(postings.doc(), 1);
+ EXPECT_EQ(postings.norm(), 1);
+ EXPECT_EQ(postings.max_block_norm(), 1);
+ EXPECT_GE(postings.block_max_score(), similarity->score(1.0F, 1));
+ EXPECT_EQ(postings.advance(), 3);
+ EXPECT_EQ(postings.norm(), 1);
+}
+
+TEST_F(SegmentPostingsTest, KeywordNormSurvivesPartitionSwitch) {
+ std::vector<ClucenePostingsCursor::Partition> parts;
+ for (uint32_t base : {0, 10}) {
+ parts.push_back({.iter = TermDocsPtr(new KeywordTermDocs({1}, {1},
{0}, 1)),
+ .begin = base,
+ .end = base + 10,
+ .has_norms = false});
+ }
+ SegmentPostings
postings(std::make_unique<ClucenePostingsCursor>(std::move(parts)), true,
+ nullptr);
+ EXPECT_EQ(postings.doc(), 1);
+ EXPECT_EQ(postings.norm(), 1);
+ EXPECT_EQ(postings.advance(), 11);
+ EXPECT_EQ(postings.norm(), 1);
+ EXPECT_EQ(postings.max_block_norm(), 1);
+}
+
TEST_F(SegmentPostingsTest, test_segment_postings_base_constructor_next_false)
{
TermDocsPtr ptr(new MockTermDocs({}, {}, {}, 0));
SegmentPostings
base(std::make_unique<ClucenePostingsCursor>(std::move(ptr)), true, nullptr);
diff --git a/be/test/storage/index/inverted/query/spi/fake_index_source.h
b/be/test/storage/index/inverted/query/spi/fake_index_source.h
index db6fcbdac4b..7a53415f6c2 100644
--- a/be/test/storage/index/inverted/query/spi/fake_index_source.h
+++ b/be/test/storage/index/inverted/query/spi/fake_index_source.h
@@ -217,18 +217,29 @@ public:
void set_scoring_stats(IndexScoringStats stats) { _scoring_stats = stats; }
+ uint32_t doc_count() const override { return _doc_count; }
+ bool batches_reads() const override { return batches; }
+
Status scoring_stats(IndexScoringStats* out) const override {
- // Most fixtures do not score expansions and deliberately provide no
source statistics.
- if (!_scoring_stats.has_value()) {
- return IndexSource::scoring_stats(out);
+ // Expansion fixtures can override statistics; conjunction fixtures
derive the source's
+ // posting shape below so docs-only terms do not enter positional
batch scoring.
+ if (_scoring_stats.has_value()) {
+ *out = *_scoring_stats;
+ return Status::OK();
+ }
+ uint64_t tokens = 0;
+ for (const auto& [term, postings] : _terms) {
+ for (const auto& posting : postings) {
+ tokens += std::max<size_t>(1, posting.positions.size());
+ }
}
- *out = *_scoring_stats;
+ *out = {.doc_count = _doc_count,
+ .token_count = tokens,
+ .has_positions = has_positions,
+ .has_norms = !norms.empty()};
return Status::OK();
}
- uint32_t doc_count() const override { return _doc_count; }
- bool batches_reads() const override { return batches; }
-
Status prepare_terms(std::span<const std::string> terms) override {
prepared.emplace_back(terms.begin(), terms.end());
return Status::OK();
@@ -236,6 +247,10 @@ public:
Status open_term(std::string_view term, bool positions, bool scoring,
std::unique_ptr<PostingsCursor>* out) override {
+ // Model a docs-only keyword source so positional scoring requests
fail in tests.
+ if (positions && !has_positions) {
+ return Status::NotSupported("fake: the index holds no positions");
+ }
opened.emplace_back(term);
*out = _cursor(term, positions, scoring);
return Status::OK();
@@ -243,6 +258,10 @@ public:
Status open_terms(std::span<const std::string> terms, bool positions, bool
scoring,
std::vector<std::unique_ptr<PostingsCursor>>* out)
override {
+ // Model a docs-only keyword source so positional scoring requests
fail in tests.
+ if (positions && !has_positions) {
+ return Status::NotSupported("fake: the index holds no positions");
+ }
opened_together.emplace_back(terms.begin(), terms.end());
out->clear();
for (const std::string& term : terms) {
@@ -304,6 +323,7 @@ public:
bool is_live(uint32_t doc) const override { return
!_deleted.contains(doc); }
bool batches = false;
+ bool has_positions = true;
// The decode work per document every cursor reports for its positions.
uint64_t position_work = 0;
// What every expansion returns before it enumerates.
diff --git
a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
index 6d00d6da478..c5c454bd0b3 100644
--- a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
+++ b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
@@ -667,6 +667,7 @@ protected:
uint64_t sum_total_term_freq = 0;
bool has_positions = true;
bool has_norms = true;
+ bool is_analyzed = true;
bool single_term_normalizer = false;
};
@@ -676,8 +677,8 @@ protected:
for (const auto& field : fields) {
RETURN_IF_ERROR(statistics->admit_snii_scoring_segment(
field.field_name, field.index_doc_count,
field.sum_total_term_freq,
- field.has_positions, field.has_norms,
field.single_term_normalizer,
- segment_accumulator));
+ field.has_positions, field.has_norms, segment_accumulator,
field.is_analyzed,
+ field.single_term_normalizer));
}
return Status::OK();
}
@@ -975,26 +976,45 @@ TEST_F(CollectionStatisticsTest,
LegacyV3RejectsSegmentWrittenWithoutNorms) {
expect_no_collected_tokens(L"1");
}
-// An index that is not analyzed never writes norms, and its scoring
statistics are collected as
-// before.
-TEST_F(CollectionStatisticsTest,
LegacyV3KeywordIndexWithoutNormsIsStillCollected) {
+TEST_F(CollectionStatisticsTest,
LegacyV3KeywordIndexWithoutNormsCollectsScoring) {
auto tablet_schema = create_legacy_v3_schema({});
const std::string segment_path = test_dir_ + "/legacy_v3_keyword_0.dat";
ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, segment_path).ok());
+ const std::string second_path = test_dir_ + "/legacy_v3_keyword_1.dat";
+ ASSERT_TRUE(write_legacy_v3_segment(tablet_schema, second_path).ok());
+
auto rowset_meta =
std::make_shared<collection_statistics::MockRowsetMeta>();
auto rowset =
std::make_shared<collection_statistics::MockRowset>(tablet_schema, rowset_meta);
- rowset->set_num_segments(1);
+ rowset->set_num_segments(2);
rowset->set_segment_path(0, segment_path);
+ rowset->set_segment_path(1, second_path);
auto reader =
std::make_shared<collection_statistics::MockRowsetReader>(rowset);
std::vector<RowSetSplits> splits {RowSetSplits(reader)};
- const Status status = stats_->collect(runtime_state_.get(), splits,
tablet_schema,
- create_search_contexts("TERM",
"alpha beta"), nullptr);
+ for (const auto* clause_type : {"TERM", "PHRASE", "EXACT"}) {
+ const Status status =
+ stats_->collect(runtime_state_.get(), splits, tablet_schema,
+ create_search_contexts(clause_type, "alpha
beta"), nullptr);
- ASSERT_TRUE(status.ok()) << status;
- expect_collected_stats(L"1", 1, 0);
- expect_collected_term(L"1", L"alpha beta", 1);
+ ASSERT_TRUE(status.ok()) << status;
+ expect_collected_stats(L"1", 2, 2);
+ expect_collected_term(L"1", L"alpha beta", 2);
+ EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 1.0F);
+ }
+}
+
+TEST_F(CollectionStatisticsTest,
SniiKeywordStatsAllowMissingPositionsAndNorms) {
+ auto resolved = resolve_snii_scoring_segment(3, 3, false, false, false);
+ ASSERT_TRUE(resolved.has_value()) << resolved.error();
+ EXPECT_EQ(resolved->doc_count, 3);
+ EXPECT_EQ(resolved->token_count, 3);
+ SniiScoringFieldInput input(L"1", 3, 3, false);
+ input.has_positions = false;
+ input.is_analyzed = false;
+ ASSERT_TRUE(admit_snii_fields_for_test(stats_.get(), {input}).ok());
+ expect_collected_stats(L"1", 3, 3);
+ EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 1.0F);
}
TEST_F(CollectionStatisticsTest, SniiScoringUsesPhysicalStatistics) {
@@ -1230,9 +1250,10 @@ TEST(CollectionStatisticsSniiScoringTest,
ResolveRejectsSegmentWithoutPositions)
}
TEST(CollectionStatisticsSniiScoringTest,
ResolveAcceptsSingleTermNormalizerWithoutPositions) {
- auto result = resolve_snii_scoring_segment(3, 2, /*has_positions=*/false,
- /*has_norms=*/false,
-
/*single_term_normalizer=*/true);
+ auto result =
+ resolve_snii_scoring_segment(3, 2, /*has_positions=*/false,
+ /*has_norms=*/false,
+ /*is_analyzed=*/true,
/*single_term_normalizer=*/true);
ASSERT_TRUE(result.has_value()) << result.error();
EXPECT_EQ(result->doc_count, 3U);
@@ -1939,7 +1960,9 @@ TEST_F(CollectionStatisticsTest,
CollectSkipsIndexWithoutAnalyzer) {
index_pb.set_index_name("kw_idx");
index_pb.set_index_type(IndexType::INVERTED);
index_pb.add_col_unique_id(kColUid);
- // No "parser" property -> should_analyzer returns false
+ // An explicit keyword parser inherits phrase support but is still not
analyzed.
+ (*index_pb.mutable_properties())["parser"] = "none";
+ (*index_pb.mutable_properties())["support_phrase"] = "true";
TabletIndex index;
index.init_from_pb(index_pb);
tablet_schema->append_index(std::move(index));
@@ -2377,7 +2400,7 @@ TEST_F(CollectionStatisticsTest,
SearchArrayStringSelectsFulltextLeafIndex) {
EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 20);
}
-TEST_F(CollectionStatisticsTest, SearchExactIgnoresAnalyzedBindingHint) {
+TEST_F(CollectionStatisticsTest,
SearchExactUsesKeywordIndexWithAnalyzedBindingHint) {
auto tablet_schema =
create_tablet_schema_with_keyword_and_fulltext_indexes();
TSearchClause clause;
clause.clause_type = "EXACT";
@@ -2397,8 +2420,8 @@ TEST_F(CollectionStatisticsTest,
SearchExactIgnoresAnalyzedBindingHint) {
runtime_state_.get(), create_search_contexts(std::move(clause),
{std::move(binding)}),
tablet_schema, &collect_infos);
- ASSERT_TRUE(status.ok()) << status.msg();
- ASSERT_EQ(collect_infos.size(), 1u);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_EQ(collect_infos.size(), 1U);
ASSERT_NE(collect_infos.begin()->second.index_meta, nullptr);
EXPECT_EQ(collect_infos.begin()->second.index_meta->index_id(), 10);
EXPECT_EQ(collect_infos.begin()->second.unique_terms,
diff --git a/be/test/storage/segment/index_reader_helper_test.cpp
b/be/test/storage/segment/index_reader_helper_test.cpp
index b7a6da09eda..64d0c173858 100644
--- a/be/test/storage/segment/index_reader_helper_test.cpp
+++ b/be/test/storage/segment/index_reader_helper_test.cpp
@@ -22,6 +22,7 @@
#include <map>
#include <memory>
#include <string>
+#include <vector>
#include "common/be_mock_util.h"
#include "storage/index/index_reader.h"
@@ -380,6 +381,7 @@ TEST_F(IndexReaderHelperTest,
IsNeedSimilarityScoreWithInvertedIndexQueryTypeTes
index_meta_pb->add_col_unique_id(1);
auto* properties = index_meta_pb->mutable_properties();
+ (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH;
(*properties)[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] =
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
@@ -448,6 +450,7 @@ TEST_F(IndexReaderHelperTest,
IsNeedSimilarityScoreWithTExprOpcodeTest) {
index_meta_pb->add_col_unique_id(1);
auto* properties = index_meta_pb->mutable_properties();
+ (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH;
(*properties)[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] =
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
@@ -460,3 +463,51 @@ TEST_F(IndexReaderHelperTest,
IsNeedSimilarityScoreWithTExprOpcodeTest) {
EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::EQ,
&index_meta));
EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::LT,
&index_meta));
}
+
+TEST_F(IndexReaderHelperTest, MatchScoringRequiresAnalyzedPhraseIndex) {
+ struct Case {
+ std::map<std::string, std::string> properties;
+ bool expected;
+ };
+ for (const auto& test : std::vector<Case> {
+ {.properties = {{"support_phrase", "true"}}, .expected =
false},
+ {.properties = {{"parser", "none"}, {"support_phrase",
"true"}},
+ .expected = false},
+ {.properties = {{"parser", "unknown"}, {"support_phrase",
"true"}},
+ .expected = false},
+ {.properties = {{"normalizer", "lowercase"},
{"support_phrase", "true"}},
+ .expected = true},
+ {.properties = {{"parser", "english"}, {"support_phrase",
"true"}},
+ .expected = true},
+ {.properties = {{"analyzer", "custom_analyzer"},
{"support_phrase", "true"}},
+ .expected = true},
+ {.properties = {{"parser", "english"}}, .expected = false},
+ {.properties = {{"parser", "english"}, {"support_phrase",
"false"}},
+ .expected = false}}) {
+ // The canonical analyzer predicate also recognizes named normalizers.
+ SCOPED_TRACE(::testing::PrintToString(test.properties));
+ TabletIndexPB pb;
+ pb.set_index_id(1);
+ pb.set_index_name("scoring_index");
+ pb.set_index_type(IndexType::INVERTED);
+ pb.add_col_unique_id(1);
+ for (const auto& [key, value] : test.properties) {
+ (*pb.mutable_properties())[key] = value;
+ }
+ TabletIndex index;
+ index.init_from_pb(pb);
+ for (auto op : {TExprOpcode::MATCH_ANY, TExprOpcode::MATCH_ALL,
TExprOpcode::MATCH_PHRASE,
+ TExprOpcode::MATCH_PHRASE_PREFIX}) {
+ EXPECT_EQ(IndexReaderHelper::is_need_similarity_score(op, &index),
test.expected);
+ }
+ for (auto type :
+ {InvertedIndexQueryType::MATCH_ANY_QUERY,
InvertedIndexQueryType::MATCH_ALL_QUERY,
+ InvertedIndexQueryType::MATCH_PHRASE_QUERY,
+ InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY}) {
+ EXPECT_EQ(IndexReaderHelper::is_need_similarity_score(type,
&index), test.expected);
+ }
+
EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::EQ,
&index));
+ EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(
+ InvertedIndexQueryType::EQUAL_QUERY, &index));
+ }
+}
diff --git a/regression-test/data/search/test_search_score_keyword_filters.out
b/regression-test/data/search/test_search_score_keyword_filters.out
new file mode 100644
index 00000000000..5b1d6730b55
--- /dev/null
+++ b/regression-test/data/search/test_search_score_keyword_filters.out
@@ -0,0 +1,240 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_false_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !V2_false_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_false_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_false_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_false_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_true_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !V2_true_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_true_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_true_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !V2_true_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_false_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !explicit_none_V2_false_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_false_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_false_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_false_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_true_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !explicit_none_V2_true_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_true_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_true_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V2_true_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_false_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !V3_false_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_false_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_false_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_false_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_true_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !V3_true_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_true_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_true_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !V3_true_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_false_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !explicit_none_V3_false_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_false_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_false_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_false_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_true_internal_and --
+1 0.983863 true
+4 0.961783 true
+
+-- !explicit_none_V3_true_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_true_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_true_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_V3_true_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_false_internal_and --
+1 0.93144 true
+4 0.90936 true
+
+-- !SNII_false_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_false_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_false_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_false_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_true_internal_and --
+1 0.93144 true
+4 0.90936 true
+
+-- !SNII_true_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_true_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_true_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !SNII_true_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_false_internal_and --
+1 0.93144 true
+4 0.90936 true
+
+-- !explicit_none_SNII_false_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_false_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_false_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_false_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_true_internal_and --
+1 0.93144 true
+4 0.90936 true
+
+-- !explicit_none_SNII_true_baseline --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_true_equality --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_true_keyword_match --
+1 0.290716 true
+4 0.268636 true
+
+-- !explicit_none_SNII_true_keyword_match_all --
+1 0.290716 true
+4 0.268636 true
diff --git a/regression-test/data/search/test_search_score_raw_index.out
b/regression-test/data/search/test_search_score_raw_index.out
new file mode 100644
index 00000000000..97acb731c06
--- /dev/null
+++ b/regression-test/data/search/test_search_score_raw_index.out
@@ -0,0 +1,205 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_unscored --
+1
+3
+9
+
+-- !V2_true_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_true_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_true_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_true_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_true_top_one --
+6 2.079442 true
+
+-- !V2_true_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_true_case --
+6 2.079442 true
+
+-- !V2_true_missing --
+
+-- !V2_false_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_false_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_false_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_false_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V2_false_top_one --
+6 2.079442 true
+
+-- !V2_false_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V2_false_case --
+6 2.079442 true
+
+-- !V2_false_missing --
+
+-- !V2_dual_exact --
+1 0.980829 true
+
+-- !V3_unscored --
+1
+3
+9
+
+-- !V3_true_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_true_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_true_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_true_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_true_top_one --
+6 2.079442 true
+
+-- !V3_true_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_true_case --
+6 2.079442 true
+
+-- !V3_true_missing --
+
+-- !V3_false_term --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_false_phrase --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_false_exact --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_false_or_top_two --
+5 1.568616 true
+10 1.568616 true
+
+-- !V3_false_top_one --
+6 2.079442 true
+
+-- !V3_false_ascending --
+1 1.232144 true
+3 1.232144 true
+9 1.232144 true
+
+-- !V3_false_case --
+6 2.079442 true
+
+-- !V3_false_missing --
+
+-- !V3_dual_exact --
+1 0.980829 true
+
+-- !SNII_unscored --
+1
+3
+9
+
+-- !SNII_true_term --
+1 1.183719 true
+3 1.183719 true
+9 1.183719 true
+
+-- !SNII_true_phrase --
+5 1.506967 true
+10 1.506967 true
+
+-- !SNII_true_exact --
+5 1.506967 true
+10 1.506967 true
+
+-- !SNII_true_or_top_two --
+5 1.506967 true
+10 1.506967 true
+
+-- !SNII_true_top_one --
+6 1.997717 true
+
+-- !SNII_true_ascending --
+1 1.183719 true
+3 1.183719 true
+9 1.183719 true
+
+-- !SNII_true_case --
+6 1.997717 true
+
+-- !SNII_true_missing --
+
+-- !SNII_false_term --
+1 1.183719 true
+3 1.183719 true
+9 1.183719 true
+
+-- !SNII_false_phrase --
+5 1.506967 true
+10 1.506967 true
+
+-- !SNII_false_exact --
+5 1.506967 true
+10 1.506967 true
+
+-- !SNII_false_or_top_two --
+5 1.506967 true
+10 1.506967 true
+
+-- !SNII_false_top_one --
+6 1.997717 true
+
+-- !SNII_false_ascending --
+1 1.183719 true
+3 1.183719 true
+9 1.183719 true
+
+-- !SNII_false_case --
+6 1.997717 true
+
+-- !SNII_false_missing --
+
+-- !SNII_dual_exact --
+1 0.980829 true
+
diff --git
a/regression-test/suites/search/test_search_score_keyword_filters.groovy
b/regression-test/suites/search/test_search_score_keyword_filters.groovy
new file mode 100644
index 00000000000..a1d41ab4861
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_keyword_filters.groovy
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_keyword_filters", "p0") {
+ // SEARCH needs index execution even when CI fuzzes these session defaults.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ for (def storageFormat : ["V2", "V3", "SNII"]) {
+ for (def explicitNone : [false, true]) {
+ String variant = explicitNone ? "explicit_none_" : ""
+ String keywordProperties = explicitNone ? 'PROPERTIES("parser" =
"none")' : ""
+ sql "DROP TABLE IF EXISTS test_search_score_keyword_filters"
+ sql """
+ CREATE TABLE test_search_score_keyword_filters (
+ id INT NOT NULL,
+ body STRING NULL,
+ tag STRING NULL,
+ INDEX idx_body(body) USING INVERTED
+ PROPERTIES("parser" = "english", "support_phrase" =
"true"),
+ INDEX idx_tag(tag) USING INVERTED ${keywordProperties}
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}"
+ )
+ """
+ sql """INSERT INTO test_search_score_keyword_filters VALUES
+ (1, 'apple apple', 'x'), (2, 'apple pear', 'y'),
+ (3, 'pear', 'x'), (4, 'apple', 'x'),
+ (5, 'apple', 'z'), (6, 'apple', NULL)
+ """
+ sql "sync"
+
+ for (def wand : [false, true]) {
+ // Inside SEARCH both fields contribute scores; keyword terms
have unit frequency.
+ quickTest("${variant}${storageFormat}_${wand}_internal_and",
"""
+ SELECT id, ROUND(bm25_score, 6), bm25_score > 0 AND
bm25_score < 1000000
+ FROM (
+ SELECT
/*+SET_VAR(enable_inverted_index_wand_query=${wand}) */
+ id, score() AS bm25_score
+ FROM test_search_score_keyword_filters
+ WHERE SEARCH('body:apple AND tag:x',
'{"mode":"standard"}')
+ ORDER BY bm25_score DESC LIMIT 10
+ ) hits
+ ORDER BY id
+ """)
+ for (def filter : [
+ ["baseline", "id IN (1, 4)"],
+ ["equality", "tag = 'x'"],
+ ["keyword_match", "tag MATCH_ANY 'x'"],
+ ["keyword_match_all", "tag MATCH_ALL 'x'"]]) {
+ // The keyword predicate filters rows; only the SEARCH
leaf contributes a score.
+
quickTest("${variant}${storageFormat}_${wand}_${filter[0]}", """
+ SELECT id, ROUND(bm25_score, 6), bm25_score > 0 AND
bm25_score < 1000000
+ FROM (
+ SELECT
/*+SET_VAR(enable_inverted_index_wand_query=${wand}) */
+ id, score() AS bm25_score
+ FROM test_search_score_keyword_filters
+ WHERE SEARCH('body:apple') AND ${filter[1]}
+ ORDER BY bm25_score DESC LIMIT 10
+ ) hits
+ ORDER BY id
+ """)
+ }
+ }
+ }
+ }
+}
diff --git a/regression-test/suites/search/test_search_score_raw_index.groovy
b/regression-test/suites/search/test_search_score_raw_index.groovy
new file mode 100644
index 00000000000..a2f0f7fc653
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_raw_index.groovy
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_raw_index", "p0") {
+ // SEARCH needs index execution even when CI fuzzes these session defaults.
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ // Score TopN pushdown accepts one ordering key. Sort ties outside the
bounded query.
+ def scoredQuery = { hint, table, predicate, direction, limit ->
+ """
+ SELECT id, ROUND(bm25_score, 6), bm25_score > 0 AND bm25_score <
1000000
+ FROM (
+ SELECT ${hint} id, score() AS bm25_score FROM ${table}
+ WHERE ${predicate}
+ ORDER BY bm25_score ${direction} LIMIT ${limit}
+ ) hits
+ ORDER BY bm25_score ${direction}, id
+ """
+ }
+ for (def storageFormat : ["V2", "V3", "SNII"]) {
+ sql "DROP TABLE IF EXISTS test_search_score_raw_index"
+ sql """
+ CREATE TABLE test_search_score_raw_index (
+ id INT NOT NULL,
+ raw_index_content STRING NULL,
+ INDEX idx_raw_content(raw_index_content) USING INVERTED
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}"
+ )
+ """
+ // Separate inserts exercise statistics across multiple segments.
+ sql """INSERT INTO test_search_score_raw_index VALUES
+ (1, 'apple'), (2, 'banana'), (3, 'apple'), (4, 'orange'),
+ (5, 'apple banana'), (6, 'Apple'), (7, NULL), (8, '')
+ """
+ sql "sync"
+ sql """INSERT INTO test_search_score_raw_index VALUES
+ (9, 'apple'), (10, 'apple banana'), (11, 'banana')
+ """
+ sql "sync"
+
+ def term = "search('raw_index_content:apple', '{\"mode\":\"lucene\"}')"
+ def phrase = "search('raw_index_content:\"apple banana\"',
'{\"mode\":\"lucene\"}')"
+ def exact = "search('raw_index_content:EXACT(apple banana)',
'{\"mode\":\"standard\"}')"
+ def rankedOr = "search('raw_index_content:apple OR
raw_index_content:\"apple banana\"', '{\"mode\":\"lucene\"}')"
+ def uniqueTop = "search('raw_index_content:apple OR
raw_index_content:Apple', '{\"mode\":\"lucene\"}')"
+ def caseSensitive = "search('raw_index_content:Apple',
'{\"mode\":\"lucene\"}')"
+ def missing = "search('raw_index_content:pear',
'{\"mode\":\"lucene\"}')"
+ quickTest("${storageFormat}_unscored", """
+ SELECT id FROM test_search_score_raw_index WHERE ${term} ORDER BY
id
+ """)
+ for (def wand : [true, false]) {
+ def hint = "/*+SET_VAR(enable_inverted_index_wand_query=${wand})
*/"
+ for (def entry : [
+ ["term", term, "DESC", 100],
+ ["phrase", phrase, "DESC", 100],
+ ["exact", exact, "DESC", 100],
+ ["or_top_two", rankedOr, "DESC", 2],
+ ["top_one", uniqueTop, "DESC", 1],
+ ["ascending", term, "ASC", 100],
+ ["case", caseSensitive, "DESC", 100],
+ ["missing", missing, "DESC", 100]]) {
+ quickTest("${storageFormat}_${wand}_${entry[0]}",
+ scoredQuery(hint, "test_search_score_raw_index",
entry[1],
+ entry[2], entry[3]))
+ }
+ }
+ sql "DROP TABLE IF EXISTS test_search_score_dual_index"
+ sql """
+ CREATE TABLE test_search_score_dual_index (
+ id INT NOT NULL,
+ content STRING NULL,
+ INDEX idx_raw(content) USING INVERTED,
+ INDEX idx_analyzed(content) USING INVERTED
+ PROPERTIES("parser"="standard", "support_phrase"="true")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation"="tag.location.default:1",
+ "inverted_index_storage_format"="${storageFormat}"
+ )
+ """
+ sql """INSERT INTO test_search_score_dual_index VALUES
+ (1, 'apple banana'), (2, 'apple'), (3, 'apple banana orange')
+ """
+ sql "sync"
+ quickTest("${storageFormat}_dual_exact",
+ scoredQuery("", "test_search_score_dual_index",
+ "search('content:EXACT(apple banana)',
'{\"mode\":\"standard\"}')",
+ "DESC", 10))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
