This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 8cf932abf67 branch-4.1: [fix](be) Score regexp and wildcard search
terms with BM25 #69052 (#69269)
8cf932abf67 is described below
commit 8cf932abf67ad76816ca41f3903cfbba578792d2
Author: Jack <[email protected]>
AuthorDate: Tue Oct 20 09:25:06 2026 +0800
branch-4.1: [fix](be) Score regexp and wildcard search terms with BM25
#69052 (#69269)
### What problem does this PR solve?
Related PR: #69052
Backport regexp and wildcard SEARCH scoring to `branch-4.1`. Scored
expansions currently return constant scores. The fix uses existing
`TermScorer` instances to compute BM25 for each expanded term and sums
the matching contributions. Unscored queries retain the bitmap path.
This uses the compatible original change from
`34a123b85da6ffc4990157820211dd08462744ec` with the branch's `query_v2`
interfaces. Legacy indexes without total-term-frequency metadata retain
the unit-average-length fallback. The backport also adds independent
term-score assertions and SQL regression coverage.
### Release note
Scored SEARCH regexp and wildcard expansions use BM25 relevance scores.
### Validation
Tested head: `07de3d3acc1adeddae8ba85c6da295aa047f0b6f`, including
current `branch-4.1` base `c7f95adaaaec4bc0caeab9ee831a9009745d56d0`.
- ASAN UT: 636 tests from 36 related suites passed; zero failures,
errors or skips. The two new expansion-scoring tests fail against the
original production implementation and pass with this fix, including
equality with independently collected `TermQuery` contributions.
- Full ASAN and Release FE/BE builds completed. After synchronizing the
newer Boolean-scoring base, both BE types were rebuilt and the unchanged
FE artifacts were reused. Both binaries retain GLIBC compatibility
(maximum required GLIBC 2.10); ASAN instrumentation was verified in the
ASAN binary.
- ASAN and Release each passed 16 related SEARCH/inverted-index suites,
with zero failed suites, fatal scripts or skips. Coverage includes
scoring/cache/TopN/range predicates, Boolean branches, regexp and array
regexp, phrase/prefix/edge, custom analyzers and raw indexes.
- Each binary passed 24 direct score-comparison SQL queries across
V2/V3, standard/keyword indexes and WAND on/off. Regexp and wildcard
scores match explicit expanded-term queries. The new regression has 48
framework-generated golden cases, replayed on both binaries, including
NULL input, empty matches, ordering and unscored queries.
- Runtime logs contain no sanitizer errors. All task-owned test
instances were stopped. Full merged-tree clang-format 16 and
English-description checks passed.
Additional static-check limit: clang-tidy 16 cannot parse the
compiler-20 JNI assertions; clang-tidy 20 is blocked by an unmatched
NOLINTEND in the unchanged baseline. An overall clang-tidy pass is not
claimed.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- [x] Manual test (score-comparison matrix described above)
- Behavior changed:
- [x] Yes. Scored regexp/wildcard expansions return BM25 contributions.
- Does this need documentation?
- [x] No. No syntax or setting is added.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../query_v2/regexp_query/regexp_weight.cpp | 50 +++
.../inverted/query_v2/regexp_query/regexp_weight.h | 6 +-
.../index/inverted/query_v2/regexp_query_test.cpp | 61 +++-
.../inverted/query_v2/wildcard_query_test.cpp | 72 ++++-
.../search/test_search_score_regexp_wildcard.out | 337 +++++++++++++++++++++
.../test_search_score_regexp_wildcard.groovy | 80 +++++
6 files changed, 591 insertions(+), 15 deletions(-)
diff --git
a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
index 5ecfad0d3ae..e2576724ab4 100644
--- a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
+++ b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.cpp
@@ -34,12 +34,18 @@
#include <gen_cpp/PaloBrokerService_types.h>
#include <algorithm>
+#include <cmath>
#include <string>
+#include <vector>
#include "storage/index/inverted/query_v2/bit_set_query/bit_set_scorer.h"
#include
"storage/index/inverted/query_v2/const_score_query/const_score_scorer.h"
#include "storage/index/inverted/query_v2/nullable_scorer.h"
+#include "storage/index/inverted/query_v2/score_combiner.h"
#include "storage/index/inverted/query_v2/segment_postings.h"
+#include "storage/index/inverted/query_v2/term_query/term_scorer.h"
+#include "storage/index/inverted/query_v2/union/buffered_union.h"
+#include "storage/index/inverted/similarity/bm25_similarity.h"
#include "storage/index/inverted/util/string_helper.h"
#include "util/hyperscan_util.h"
@@ -110,6 +116,10 @@ ScorerPtr RegexpWeight::regexp_scorer(const
QueryExecutionContext& context,
}
auto reader = lookup_reader(_field, context, binding_key);
+ if (_enable_scoring) {
+ return scored_terms_scorer(reader.get(), context, binding_key,
matching_terms);
+ }
+
auto doc_bitset = std::make_shared<roaring::Roaring>();
for (const auto& term : matching_terms) {
auto segment_postings =
@@ -126,6 +136,46 @@ ScorerPtr RegexpWeight::regexp_scorer(const
QueryExecutionContext& context,
return const_score;
}
+ScorerPtr RegexpWeight::scored_terms_scorer(lucene::index::IndexReader* reader,
+ const QueryExecutionContext&
context,
+ const std::string& binding_key,
+ const std::vector<std::wstring>&
matching_terms) {
+ // Expanded terms are not collected in collection statistics, so use
reader statistics.
+ const auto num_docs = reader->maxDoc();
+ DORIS_CHECK_GT(num_docs, 0);
+ const auto total_term_freq = reader->sumTotalTermFreq(_field.c_str());
+ // Without norms, postings use a document length of one.
+ const float avgdl = static_cast<float>(total_term_freq.value_or(num_docs))
/ num_docs;
+ DORIS_CHECK_GT(avgdl, 0.0F);
+
+ std::vector<ScorerPtr> term_scorers;
+ term_scorers.reserve(matching_terms.size());
+ auto logical_field = logical_field_or_fallback(context, binding_key,
_field);
+ for (const auto& term : matching_terms) {
+ Term lucene_term(_field.c_str(), term.c_str());
+ const int32_t doc_freq = reader->docFreq(&lucene_term);
+ if (doc_freq == 0) {
+ continue;
+ }
+ const auto idf = static_cast<float>(
+ std::log1p((static_cast<double>(num_docs) - doc_freq + 0.5) /
(doc_freq + 0.5)));
+ auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
+ auto posting =
+ create_term_posting(reader, _field, term, true, similarity,
_context->io_ctx);
+ if (posting) {
+ term_scorers.emplace_back(std::make_shared<TermScorer>(
+ std::move(posting), std::move(similarity), logical_field));
+ }
+ }
+ if (term_scorers.empty()) {
+ return std::make_shared<EmptyScorer>();
+ }
+ if (term_scorers.size() == 1) {
+ return std::move(term_scorers.front());
+ }
+ return make_buffered_union(term_scorers, std::make_shared<SumCombiner>());
+}
+
std::optional<std::string> RegexpWeight::get_regex_prefix(const std::string&
pattern) {
DBUG_EXECUTE_IF("RegexpQuery.get_regex_prefix", { return std::nullopt; });
diff --git
a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
index f8b1a0ba9fa..6ef13c71fa5 100644
--- a/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
+++ b/be/src/storage/index/inverted/query_v2/regexp_query/regexp_weight.h
@@ -35,6 +35,10 @@ public:
private:
ScorerPtr regexp_scorer(const QueryExecutionContext& context, const
std::string& binding_key);
+ ScorerPtr scored_terms_scorer(lucene::index::IndexReader* reader,
+ const QueryExecutionContext& context,
+ const std::string& binding_key,
+ const std::vector<std::wstring>&
matching_terms);
std::optional<std::string> get_regex_prefix(const std::string& pattern);
void collect_matching_terms(const QueryExecutionContext& context,
@@ -46,7 +50,7 @@ private:
std::wstring _field;
std::string _pattern;
- [[maybe_unused]] bool _enable_scoring = false;
+ bool _enable_scoring = false;
bool _nullable = true;
// Set to 0 to disable limit (ES has no default limit for prefix queries)
// The limit prevents collecting too many terms, but can cause incorrect
results
diff --git a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
index aea4ed627ed..ed02ac578d3 100644
--- a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
+++ b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp
@@ -28,6 +28,7 @@
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
#include "storage/index/inverted/query_v2/regexp_query/regexp_weight.h"
+#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/util/string_helper.h"
CL_NS_USE(search)
@@ -58,10 +59,13 @@ public:
private:
void create_test_index(const std::string& field_name, const std::string&
dir) {
std::vector<std::string> test_data = {
- "apple123", "apple456", "banana789", "test123abc",
"pattern456",
- "regex999", "match123", "search456", "prefix123",
"suffix789",
- "apple_test", "banana_data", "test_pattern", "data_regex",
"apple_banana",
- "test_match", "pattern_test", "prefix_suffix", "abc123xyz",
"def456ghi"};
+ "apple123", "apple456", "banana789",
+ "test123abc", "pattern456", "regex999",
+ "match123", "search456", "prefix123",
+ "suffix789", "apple_test", "banana_data",
+ "test_pattern", "data_regex", "apple_banana",
+ "test_match", "pattern_test", "prefix_suffix",
+ "abc123xyz", "def456ghi", "apple789 apple789 apple456"};
CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
@@ -167,6 +171,53 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
ASSERT_NE(regexp_weight, nullptr);
}
+TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
+ auto context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics = std::make_shared<CollectionStatistics>();
+ context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+ auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+ auto reader_holder =
make_shared_reader(lucene::index::IndexReader::open(dir, true));
+ ASSERT_NE(reader_holder, nullptr);
+
+ std::wstring field = StringHelper::to_wstring("content");
+ auto query = std::make_shared<query_v2::RegexpQuery>(context, field,
"apple.*");
+ auto weight = query->weight(true);
+
+ query_v2::QueryExecutionContext exec_ctx;
+ exec_ctx.segment_num_rows = reader_holder->maxDoc();
+ exec_ctx.readers = {reader_holder};
+ exec_ctx.field_reader_bindings.emplace(field, reader_holder);
+
+ auto statistics = context->collection_statistics;
+ statistics->_total_num_docs = reader_holder->maxDoc();
+ statistics->_total_num_tokens[field] =
+
reader_holder->sumTotalTermFreq(field.c_str()).value_or(reader_holder->maxDoc());
+ for (const auto* term : {L"apple123", L"apple789", L"apple456"}) {
+ Term lucene_term(field.c_str(), term);
+ statistics->_term_doc_freqs[field][term] =
reader_holder->docFreq(&lucene_term);
+ }
+ auto term_score = [&](const wchar_t* term, uint32_t doc_id) {
+ auto term_query = std::make_shared<query_v2::TermQuery>(context,
field, term);
+ auto term_scorer = term_query->weight(true)->scorer(exec_ctx);
+ EXPECT_EQ(term_scorer->seek(doc_id), doc_id);
+ return term_scorer->score();
+ };
+
+ auto scorer = weight->scorer(exec_ctx);
+ ASSERT_EQ(scorer->seek(0), 0);
+ float single_term_score = scorer->score();
+ EXPECT_NE(single_term_score, 1.0F);
+ EXPECT_FLOAT_EQ(single_term_score, term_score(L"apple123", 0));
+ uint32_t last_doc = reader_holder->maxDoc() - 1;
+ ASSERT_EQ(scorer->seek(last_doc), last_doc);
+ EXPECT_GT(scorer->score(), single_term_score);
+ EXPECT_NEAR(scorer->score(),
+ term_score(L"apple789", last_doc) + term_score(L"apple456",
last_doc), 1e-5F);
+
+ _CLDECDELETE(dir);
+}
+
// Test regexp query execution
TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
auto context = std::make_shared<IndexQueryContext>();
@@ -574,7 +625,7 @@ TEST_F(RegexpQueryV2Test,
test_make_exact_match_wildcard_pattern) {
doc = scorer->advance();
}
- EXPECT_EQ(result.cardinality(), 20);
+ EXPECT_EQ(result.cardinality(), 21);
_CLDECDELETE(dir);
}
diff --git a/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
b/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
index af119cdd260..3807a2787db 100644
--- a/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
+++ b/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp
@@ -27,6 +27,7 @@
#include "common/status.h"
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
+#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/query_v2/wildcard_query/wildcard_weight.h"
#include "storage/index/inverted/util/string_helper.h"
@@ -57,14 +58,20 @@ public:
private:
void create_test_index(const std::string& field_name, const std::string&
dir) {
- std::vector<std::string> test_data = {
- "apple", "application", "apply", "apricot",
"banana", "band",
- "bandana", "can", "candy", "cat",
"catalog", "dog",
- "document", "door", "hello", "help",
"helpful", "test",
- "testing", "testcase", "wildcard", "wild",
"wilderness", "card",
- "cardboard", "abc", "abcd", "abcde",
"prefix123", "123suffix",
- "pre123", "123suf", "both123both", "star", "start",
"started",
- "starter", "question", "quest", "query",
"queries"};
+ std::vector<std::string> test_data = {"apple", "application",
"apply",
+ "apricot", "banana",
"band",
+ "bandana", "can",
"candy",
+ "cat", "catalog",
"dog",
+ "document", "door",
"hello",
+ "help", "helpful",
"test",
+ "testing", "testcase",
"wildcard",
+ "wild", "wilderness",
"card",
+ "cardboard", "abc",
"abcd",
+ "abcde", "prefix123",
"123suffix",
+ "pre123", "123suf",
"both123both",
+ "star", "start",
"started",
+ "starter", "question",
"quest",
+ "query", "queries",
"test test testing"};
CustomAnalyzerConfig::Builder builder;
builder.with_tokenizer_config("standard", {});
@@ -155,6 +162,53 @@ TEST_F(WildcardQueryV2Test,
test_wildcard_query_with_scoring) {
ASSERT_NE(wildcard_weight, nullptr);
}
+TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) {
+ auto context = std::make_shared<IndexQueryContext>();
+ context->collection_statistics = std::make_shared<CollectionStatistics>();
+ context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+ auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+ auto reader_holder =
make_shared_reader(lucene::index::IndexReader::open(dir, true));
+ ASSERT_NE(reader_holder, nullptr);
+
+ std::wstring field = StringHelper::to_wstring("field");
+ auto query = std::make_shared<query_v2::WildcardQuery>(context, field,
"test*");
+ auto weight = query->weight(true);
+
+ query_v2::QueryExecutionContext exec_ctx;
+ exec_ctx.segment_num_rows = reader_holder->maxDoc();
+ exec_ctx.readers = {reader_holder};
+ exec_ctx.field_reader_bindings.emplace(field, reader_holder);
+
+ auto statistics = context->collection_statistics;
+ statistics->_total_num_docs = reader_holder->maxDoc();
+ statistics->_total_num_tokens[field] =
+
reader_holder->sumTotalTermFreq(field.c_str()).value_or(reader_holder->maxDoc());
+ for (const auto* term : {L"test", L"testing"}) {
+ Term lucene_term(field.c_str(), term);
+ statistics->_term_doc_freqs[field][term] =
reader_holder->docFreq(&lucene_term);
+ }
+ auto term_score = [&](const wchar_t* term, uint32_t doc_id) {
+ auto term_query = std::make_shared<query_v2::TermQuery>(context,
field, term);
+ auto term_scorer = term_query->weight(true)->scorer(exec_ctx);
+ EXPECT_EQ(term_scorer->seek(doc_id), doc_id);
+ return term_scorer->score();
+ };
+
+ auto scorer = weight->scorer(exec_ctx);
+ ASSERT_EQ(scorer->seek(17), 17);
+ float single_term_score = scorer->score();
+ EXPECT_NE(single_term_score, 1.0F);
+ EXPECT_FLOAT_EQ(single_term_score, term_score(L"test", 17));
+ uint32_t last_doc = reader_holder->maxDoc() - 1;
+ ASSERT_EQ(scorer->seek(last_doc), last_doc);
+ EXPECT_GT(scorer->score(), single_term_score);
+ EXPECT_NEAR(scorer->score(), term_score(L"test", last_doc) +
term_score(L"testing", last_doc),
+ 1e-5F);
+
+ _CLDECDELETE(dir);
+}
+
// Test wildcard query with asterisk prefix pattern
TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) {
auto context = std::make_shared<IndexQueryContext>();
@@ -524,4 +578,4 @@ TEST_F(WildcardQueryV2Test,
test_wildcard_query_move_semantics) {
ASSERT_NE(weight2, nullptr);
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/regression-test/data/search/test_search_score_regexp_wildcard.out
b/regression-test/data/search/test_search_score_regexp_wildcard.out
new file mode 100644
index 00000000000..393a5781dab
--- /dev/null
+++ b/regression-test/data/search/test_search_score_regexp_wildcard.out
@@ -0,0 +1,337 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !V2_standard_unscored_regexp --
+1
+2
+3
+4
+7
+8
+
+-- !V2_standard_unscored_wildcard --
+1
+2
+3
+4
+7
+8
+
+-- !V2_standard_false_regexp --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V2_standard_false_wildcard --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V2_standard_false_terms --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V2_standard_false_ascending --
+1 0.584433
+2 0.655108
+3 1.120813
+7 1.248584
+4 1.312985
+8 2.087193
+
+-- !V2_standard_false_missing --
+
+-- !V2_standard_true_regexp --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V2_standard_true_wildcard --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V2_standard_true_terms --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V2_standard_true_ascending --
+1 0.584433
+2 0.655108
+3 1.120813
+7 1.248584
+4 1.312985
+8 2.087193
+
+-- !V2_standard_true_missing --
+
+-- !V2_none_unscored_regexp --
+1
+2
+3
+4
+7
+8
+
+-- !V2_none_unscored_wildcard --
+1
+2
+3
+4
+7
+8
+
+-- !V2_none_false_regexp --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_false_wildcard --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_false_terms --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_false_ascending --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_false_missing --
+
+-- !V2_none_true_regexp --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_true_wildcard --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_true_terms --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_true_ascending --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V2_none_true_missing --
+
+-- !V3_standard_unscored_regexp --
+1
+2
+3
+4
+7
+8
+
+-- !V3_standard_unscored_wildcard --
+1
+2
+3
+4
+7
+8
+
+-- !V3_standard_false_regexp --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V3_standard_false_wildcard --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V3_standard_false_terms --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V3_standard_false_ascending --
+1 0.584433
+2 0.655108
+3 1.120813
+7 1.248584
+4 1.312985
+8 2.087193
+
+-- !V3_standard_false_missing --
+
+-- !V3_standard_true_regexp --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V3_standard_true_wildcard --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V3_standard_true_terms --
+8 2.087193
+4 1.312985
+7 1.248584
+3 1.120813
+2 0.655108
+1 0.584433
+
+-- !V3_standard_true_ascending --
+1 0.584433
+2 0.655108
+3 1.120813
+7 1.248584
+4 1.312985
+8 2.087193
+
+-- !V3_standard_true_missing --
+
+-- !V3_none_unscored_regexp --
+1
+2
+3
+4
+7
+8
+
+-- !V3_none_unscored_wildcard --
+1
+2
+3
+4
+7
+8
+
+-- !V3_none_false_regexp --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_false_wildcard --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_false_terms --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_false_ascending --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_false_missing --
+
+-- !V3_none_true_regexp --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_true_wildcard --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_true_terms --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_true_ascending --
+1 1.791759
+2 1.791759
+3 1.791759
+4 1.791759
+7 1.791759
+8 1.791759
+
+-- !V3_none_true_missing --
+
diff --git
a/regression-test/suites/search/test_search_score_regexp_wildcard.groovy
b/regression-test/suites/search/test_search_score_regexp_wildcard.groovy
new file mode 100644
index 00000000000..01cd72c70a7
--- /dev/null
+++ b/regression-test/suites/search/test_search_score_regexp_wildcard.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_search_score_regexp_wildcard", "p0") {
+ sql "set enable_common_expr_pushdown = true"
+ sql "set enable_inverted_index_query = true"
+
+ def scoredQuery = { predicate, direction, limit ->
+ """
+ SELECT id, ROUND(bm25_score, 6)
+ FROM (
+ SELECT id, score() AS bm25_score
+ FROM test_search_score_regexp_wildcard
+ WHERE ${predicate}
+ ORDER BY bm25_score ${direction} LIMIT ${limit}
+ ) hits
+ ORDER BY bm25_score ${direction}, id
+ """
+ }
+ for (String storageFormat : ["V2", "V3"]) {
+ for (String parser : ["standard", "none"]) {
+ sql "DROP TABLE IF EXISTS test_search_score_regexp_wildcard"
+ sql """
+ CREATE TABLE test_search_score_regexp_wildcard (
+ id INT NOT NULL,
+ content STRING NULL,
+ INDEX idx_content(content) USING INVERTED
PROPERTIES("parser" = "${parser}")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "inverted_index_storage_format" = "${storageFormat}"
+ )
+ """
+ sql """
+ INSERT INTO test_search_score_regexp_wildcard VALUES
+ (1, 'apple'), (2, 'apple apple apple'), (3, 'application'),
+ (4, 'apple application'), (5, 'banana'), (6, NULL),
+ (7, 'apple application apple'), (8, 'apply apple')
+ """
+ sql "sync"
+ String regexp = "search('content:/app.*/',
'{\"mode\":\"lucene\"}')"
+ String wildcard = "search('content:*pp*', '{\"mode\":\"lucene\"}')"
+ String terms = parser == "standard"
+ ? "search('content:apple OR content:application OR
content:apply', '{\"mode\":\"lucene\"}')"
+ : "search('content:EXACT(apple) OR content:EXACT(apple
apple apple) OR content:EXACT(application) OR content:EXACT(apple application)
OR content:EXACT(apple application apple) OR content:EXACT(apply apple)',
'{\"mode\":\"standard\"}')"
+ String label = "${storageFormat}_${parser}"
+ quickTest("${label}_unscored_regexp", """
+ SELECT id FROM test_search_score_regexp_wildcard WHERE
${regexp} ORDER BY id
+ """)
+ quickTest("${label}_unscored_wildcard", """
+ SELECT id FROM test_search_score_regexp_wildcard WHERE
${wildcard} ORDER BY id
+ """)
+ for (boolean wand : [false, true]) {
+ sql "set enable_inverted_index_wand_query = ${wand}"
+ quickTest("${label}_${wand}_regexp", scoredQuery(regexp,
"DESC", 100))
+ quickTest("${label}_${wand}_wildcard", scoredQuery(wildcard,
"DESC", 100))
+ quickTest("${label}_${wand}_terms", scoredQuery(terms, "DESC",
100))
+ quickTest("${label}_${wand}_ascending", scoredQuery(regexp,
"ASC", 100))
+ quickTest("${label}_${wand}_missing", scoredQuery(
+ "search('content:/missing.*/',
'{\"mode\":\"lucene\"}')", "DESC", 100))
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
