This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5cc868cf8c1 [fix](function) Advance past zero-width Boost matches in
regexp_extract_all (#68358)
5cc868cf8c1 is described below
commit 5cc868cf8c17c3017a26152f7387a478c14c41ac
Author: Jerry Hu <[email protected]>
AuthorDate: Thu Oct 15 14:07:47 2026 +0800
[fix](function) Advance past zero-width Boost matches in regexp_extract_all
(#68358)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #68358
Problem Summary: When extended regex falls back to Boost.Regex,
regexp_extract_all and regexp_extract_all_array can extract the same zero-width
capture more than once because the next search advances from the old search
start. Replace the shared Boost search loop with cregex_iterator over the
original input range. This preserves lookbehind, anchors, word boundaries, and
same-position consuming retries while enumerating zero-width matches correctly.
Add unit and regression cases for bo [...]
### Release note
Fix duplicate zero-width captures in extended-regex extraction while
preserving assertion context and same-position consuming matches.
### Check List (For Author)
- Test: Current CI compile, BE UT, FE UT, and regression checks passed.
Local isolated Boost 1.81 probes passed 17 semantic cases under ASAN/UBSAN; all
12 added regression expectations were checked; build-hygiene checks passed.
Full Doris suites were not rerun locally during review.
- Behavior changed: Yes, correct duplicate and missing captures in the
Boost extraction path.
- Does this need documentation: No, bug fix to existing function behavior.
---
be/src/exprs/function/function_regexp.cpp | 21 +++---
be/test/exprs/function/function_like_test.cpp | 78 ++++++++++++++++++++++
.../test_string_function_regexp.out | 37 ++++++++++
.../test_string_function_regexp.groovy | 15 +++++
4 files changed, 138 insertions(+), 13 deletions(-)
diff --git a/be/src/exprs/function/function_regexp.cpp
b/be/src/exprs/function/function_regexp.cpp
index 22b559a4625..c88a64ce802 100644
--- a/be/src/exprs/function/function_regexp.cpp
+++ b/be/src/exprs/function/function_regexp.cpp
@@ -288,22 +288,17 @@ struct RegexpExtractEngine {
}
}
} else if (is_boost()) {
- const char* search_start = data;
- const char* search_end = data + size;
- boost::match_results<const char*> matches;
-
- while (boost::regex_search(search_start, search_end, matches,
*boost_regex)) {
+ // Iterate over the whole input instead of re-running regex_search
on a
+ // shrinking range: the iterator keeps the preceding text visible
to
+ // lookbehind/`^`/`\b` and steps past zero-width matches (e.g.
lookahead)
+ // without matching the same position twice.
+ boost::cregex_iterator it(data, data + size, *boost_regex);
+ boost::cregex_iterator end;
+ for (; it != end; ++it) {
+ const auto& matches = *it;
if (matches.size() > 1 && matches[1].matched) {
results.emplace_back(matches[1].str());
}
- if (matches[0].length() == 0) {
- if (search_start == search_end) {
- break;
- }
- search_start += 1;
- } else {
- search_start = matches[0].second;
- }
}
}
}
diff --git a/be/test/exprs/function/function_like_test.cpp
b/be/test/exprs/function/function_like_test.cpp
index ec88452bbbe..63b51912180 100644
--- a/be/test/exprs/function/function_like_test.cpp
+++ b/be/test/exprs/function/function_like_test.cpp
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#include <gen_cpp/PaloInternalService_types.h>
+
#include <cstdint>
#include <memory>
#include <optional>
@@ -747,6 +749,82 @@ TEST(FunctionLikeTest, regexp_extract_all_array) { //
NOLINT(readability-functio
}
}
+// Patterns RE2 rejects (lookahead here) are compiled by Boost once extended
regex is
+// enabled. A zero-width Boost match must be stepped over from the matched
position,
+// otherwise the same position is matched and extracted again.
+TEST(FunctionLikeTest, regexp_extract_all_boost_zero_width_match) {
+ auto str_type = std::make_shared<DataTypeString>();
+
+ auto run_case = [&](const std::string& func_name, const DataTypePtr&
return_type,
+ const std::string& str, const std::string& pattern,
+ const std::string& expected) {
+ auto col_str = ColumnString::create();
+ col_str->insert_data(str.data(), str.size());
+ auto col_pattern = ColumnString::create();
+ col_pattern->insert_data(pattern.data(), pattern.size());
+
+ Block block;
+ block.insert({std::move(col_str), str_type, "str"});
+ block.insert({ColumnConst::create(std::move(col_pattern), 1),
str_type, "pattern"});
+ block.insert({nullptr, return_type, "result"});
+
+ ColumnsWithTypeAndName arg_cols = {block.get_by_position(0),
block.get_by_position(1)};
+ auto func =
+ SimpleFunctionFactory::instance().get_function(func_name,
arg_cols, return_type);
+ ASSERT_TRUE(func != nullptr);
+
+ std::vector<DataTypePtr> arg_types = {str_type, str_type};
+ FunctionUtils fn_utils({}, arg_types, false);
+ auto* fn_ctx = fn_utils.get_fn_ctx();
+ TQueryOptions query_options = fn_ctx->state()->query_options();
+ query_options.__set_enable_extended_regex(true);
+ fn_ctx->state()->set_query_options(query_options);
+ fn_ctx->set_constant_cols(
+ {nullptr,
std::make_shared<ColumnPtrWrapper>(block.get_by_position(1).column)});
+
+ ASSERT_EQ(Status::OK(), func->open(fn_ctx,
FunctionContext::FRAGMENT_LOCAL));
+ ASSERT_EQ(Status::OK(), func->open(fn_ctx,
FunctionContext::THREAD_LOCAL));
+ ASSERT_EQ(Status::OK(), func->execute(fn_ctx, block, {0, 1}, 2, 1));
+
+ auto result_col = block.get_by_position(2).column;
+ ASSERT_TRUE(result_col.get() != nullptr);
+ ASSERT_FALSE(result_col->is_null_at(0));
+ EXPECT_EQ(expected, return_type->to_string(*result_col, 0))
+ << func_name << " input: '" << str << "', pattern: '" <<
pattern << "'";
+
+ static_cast<void>(func->close(fn_ctx, FunctionContext::THREAD_LOCAL));
+ static_cast<void>(func->close(fn_ctx,
FunctionContext::FRAGMENT_LOCAL));
+ };
+
+ auto string_type = make_nullable(std::make_shared<DataTypeString>());
+ auto array_type = make_nullable(
+
std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeString>())));
+
+ // The lookahead matches once, at the last position; it must not be
extracted twice.
+ run_case("regexp_extract_all", string_type, "ba", "(?=(a))", "['a']");
+ run_case("regexp_extract_all_array", array_type, "ba", "(?=(a))",
R"(["a"])");
+ // Zero-width matches at distinct positions are all still found, exactly
once each.
+ run_case("regexp_extract_all", string_type, "aba", "(?=(a))", "['a','a']");
+ run_case("regexp_extract_all_array", array_type, "aba", "(?=(a))",
R"(["a", "a"])");
+ run_case("regexp_extract_all", string_type, "aaa", "(?=(a))",
"['a','a','a']");
+ run_case("regexp_extract_all_array", array_type, "aaa", "(?=(a))",
R"(["a", "a", "a"])");
+ // Non-zero-width Boost matches keep advancing past the whole match.
+ run_case("regexp_extract_all", string_type, "foo123bar456baz",
"(\\d{3})(?=bar|baz)",
+ "['123','456']");
+ run_case("regexp_extract_all_array", array_type, "foo123bar456baz",
"(\\d{3})(?=bar|baz)",
+ R"(["123", "456"])");
+ // The text before a zero-width match stays visible to lookbehind at the
next position.
+ run_case("regexp_extract_all", string_type, "aa", "(?<=(a))", "['a','a']");
+ run_case("regexp_extract_all_array", array_type, "aa", "(?<=(a))",
R"(["a", "a"])");
+ // After an empty match, a consuming match at the same position is still
tried.
+ run_case("regexp_extract_all", string_type, "xa", "((?<=x)|a)",
"['','a']");
+ run_case("regexp_extract_all_array", array_type, "xa", "((?<=x)|a)",
R"(["", "a"])");
+ // Resuming after a zero-width match must not let `^` anchor in the middle
of the input
+ // (the string form renders "no match" as an empty string).
+ run_case("regexp_extract_all", string_type, "xxa?b", "(?<=a)|^(b)", "");
+ run_case("regexp_extract_all_array", array_type, "xxa?b", "(?<=a)|^(b)",
"[]");
+}
+
TEST(FunctionLikeTest, split_by_regexp_preserves_original_anchor) {
const std::string input = "aaa";
const std::string pattern = "^a";
diff --git
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
index 91f38653d23..645da8b4e24 100644
---
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
+++
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
@@ -247,6 +247,24 @@ aXb
-- !sql_regexp_extract_all_5 --
['Case1','Case2','Case3']
+-- !sql_regexp_extract_all_zero_width_1 --
+['a']
+
+-- !sql_regexp_extract_all_zero_width_2 --
+['a','a']
+
+-- !sql_regexp_extract_all_zero_width_3 --
+['a','a','a']
+
+-- !sql_regexp_extract_all_zero_width_4 --
+['a','a']
+
+-- !sql_regexp_extract_all_zero_width_5 --
+['','a']
+
+-- !sql_regexp_extract_all_zero_width_6 --
+
+
-- !sql_regexp_extract_all_6 --
['foo\nbar']
@@ -318,6 +336,24 @@ billie eillish 1 ["billie", "eillish"]
[]
[]
+-- !regexp_extract_all_array_zero_width_1 --
+["a"]
+
+-- !regexp_extract_all_array_zero_width_2 --
+["a", "a"]
+
+-- !regexp_extract_all_array_zero_width_3 --
+["a", "a", "a"]
+
+-- !regexp_extract_all_array_zero_width_4 --
+["a", "a"]
+
+-- !regexp_extract_all_array_zero_width_5 --
+["", "a"]
+
+-- !regexp_extract_all_array_zero_width_6 --
+[]
+
-- !sql --
a-b-c
@@ -544,3 +580,4 @@ Ben
-- !sql_field5 --
2
+
diff --git
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
index b7aeaee17f3..e6fb8a35f73 100644
---
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
+++
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
@@ -120,6 +120,13 @@ suite("test_string_function_regexp") {
qt_sql_regexp_extract_all_3 'SELECT
REGEXP_EXTRACT_ALL(\'foo123bar456baz\', \'(\\\\d{3})(?=bar|baz)\');'
qt_sql_regexp_extract_all_4 'SELECT
REGEXP_EXTRACT_ALL(\'ID:AA-1,ID:BB-2,ID:CC-3\', \'(?<=ID:)([A-Z]{2}-\\\\d)\');'
qt_sql_regexp_extract_all_5 'SELECT
REGEXP_EXTRACT_ALL(\'EdgeCase1EdgeCase2EdgeCase3\',
\'(?<=Edge)(Case\\\\d)(?=Edge|$)\');'
+ // zero-width Boost matches must be extracted once per position
+ qt_sql_regexp_extract_all_zero_width_1 "SELECT REGEXP_EXTRACT_ALL('ba',
'(?=(a))');"
+ qt_sql_regexp_extract_all_zero_width_2 "SELECT REGEXP_EXTRACT_ALL('aba',
'(?=(a))');"
+ qt_sql_regexp_extract_all_zero_width_3 "SELECT REGEXP_EXTRACT_ALL('aaa',
'(?=(a))');"
+ qt_sql_regexp_extract_all_zero_width_4 "SELECT REGEXP_EXTRACT_ALL('aa',
'(?<=(a))');"
+ qt_sql_regexp_extract_all_zero_width_5 "SELECT REGEXP_EXTRACT_ALL('xa',
'((?<=x)|a)');"
+ qt_sql_regexp_extract_all_zero_width_6 "SELECT REGEXP_EXTRACT_ALL('xxa?b',
'(?<=a)|^(b)');"
sql "set enable_extended_regex = false;"
qt_sql_regexp_extract_all_6 "SELECT REGEXP_EXTRACT_ALL(concat('foo',
char(10), 'bar'), '(foo.bar)');"
@@ -143,6 +150,14 @@ suite("test_string_function_regexp") {
qt_regexp_extract_all_array_9 "SELECT regexp_extract_all_array(k,
'([a-z]+)') from test_string_function_regexp ORDER BY k;"
qt_regexp_extract_all_array_10 "SELECT k, v, regexp_extract_all_array(k,
'(\\\\w+)') from test_string_function_regexp ORDER BY k, v;"
qt_regexp_extract_all_array_11 "SELECT regexp_extract_all_array(k,
concat('^', k)) from test_string_function_regexp WHERE k IS NOT NULL ORDER BY
k;"
+ sql "set enable_extended_regex = true;"
+ qt_regexp_extract_all_array_zero_width_1 "SELECT
regexp_extract_all_array('ba', '(?=(a))');"
+ qt_regexp_extract_all_array_zero_width_2 "SELECT
regexp_extract_all_array('aba', '(?=(a))');"
+ qt_regexp_extract_all_array_zero_width_3 "SELECT
regexp_extract_all_array('aaa', '(?=(a))');"
+ qt_regexp_extract_all_array_zero_width_4 "SELECT
regexp_extract_all_array('aa', '(?<=(a))');"
+ qt_regexp_extract_all_array_zero_width_5 "SELECT
regexp_extract_all_array('xa', '((?<=x)|a)');"
+ qt_regexp_extract_all_array_zero_width_6 "SELECT
regexp_extract_all_array('xxa?b', '(?<=a)|^(b)');"
+ sql "set enable_extended_regex = false;"
qt_sql "SELECT regexp_replace('a b c', \" \", \"-\");"
qt_sql "SELECT regexp_replace('a b c','(b)','<\\\\1>');"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
