This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new daa7c620c6e branch-4.1: [fix](function) avoid ICU default-locale
contention #66440 (#67071)
daa7c620c6e is described below
commit daa7c620c6e0c275a0a3474395890a7399491ee6
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Aug 25 09:52:10 2026 +0800
branch-4.1: [fix](function) avoid ICU default-locale contention #66440
(#67071)
Cherry-picked from #66440
Co-authored-by: York Cao <[email protected]>
---
be/src/exprs/function/function_string.cpp | 7 +--
be/test/exprs/function/function_string_test.cpp | 63 ++++++++++++++++++++++++-
2 files changed, 66 insertions(+), 4 deletions(-)
diff --git a/be/src/exprs/function/function_string.cpp
b/be/src/exprs/function/function_string.cpp
index 4eb0b1462cb..4ee19bca304 100644
--- a/be/src/exprs/function/function_string.cpp
+++ b/be/src/exprs/function/function_string.cpp
@@ -18,6 +18,7 @@
#include <ctype.h>
#include <math.h>
#include <re2/stringpiece.h>
+#include <unicode/locid.h>
#include <unicode/schriter.h>
#include <unicode/uchar.h>
#include <unicode/unistr.h>
@@ -593,7 +594,7 @@ struct TransferImpl {
icu::StringPiece sp;
sp.set(data, size);
icu::UnicodeString unicode_str = icu::UnicodeString::fromUTF8(sp);
- unicode_str.toUpper();
+ unicode_str.toUpper(icu::Locale::getRoot());
unicode_str.toUTF8String(result);
}
@@ -601,7 +602,7 @@ struct TransferImpl {
icu::StringPiece sp;
sp.set(data, size);
icu::UnicodeString unicode_str = icu::UnicodeString::fromUTF8(sp);
- unicode_str.toLower();
+ unicode_str.toLower(icu::Locale::getRoot());
unicode_str.toUTF8String(result);
}
};
@@ -683,7 +684,7 @@ struct InitcapImpl {
icu::StringPiece sp;
sp.set(data, size);
icu::UnicodeString unicode_str = icu::UnicodeString::fromUTF8(sp);
- unicode_str.toLower();
+ unicode_str.toLower(icu::Locale::getRoot());
icu::UnicodeString output_str;
bool need_capitalize = true;
icu::StringCharacterIterator iter(unicode_str);
diff --git a/be/test/exprs/function/function_string_test.cpp
b/be/test/exprs/function/function_string_test.cpp
index 23bdb855704..fc10591f0ff 100644
--- a/be/test/exprs/function/function_string_test.cpp
+++ b/be/test/exprs/function/function_string_test.cpp
@@ -15,6 +15,9 @@
// specific language governing permissions and limitations
// under the License.
+#include <unicode/locid.h>
+#include <unicode/utypes.h>
+
#include <cstdint>
#include <cstring>
#include <memory>
@@ -26,6 +29,7 @@
#include "core/field.h"
#include "core/types.h"
#include "exprs/function/function_test_util.h"
+#include "util/defer_op.h"
#include "util/encryption_util.h"
#include "util/md5.h"
@@ -548,6 +552,8 @@ TEST(function_string_test, function_string_lower_test) {
{{std::string("GROSSE")}, std::string("grosse")},
{{std::string("Å")}, std::string("å")},
{{std::string("ΣΟΦΟΣ")}, std::string("σοφος")},
+ {{std::string("Iİı")}, std::string("ii̇ı")},
+ {{std::string("ΟΣ")}, std::string("ος")},
{{std::string("123ABC_")}, std::string("123abc_")},
{{std::string("MYtestSTR")}, std::string("myteststr")},
{{std::string("")}, std::string("")},
@@ -572,6 +578,7 @@ TEST(function_string_test, function_string_upper_test) {
{{std::string("MYtestSTR")}, std::string("MYTESTSTR")},
{{std::string("àç")}, std::string("ÀÇ")},
{{std::string("straße")}, std::string("STRASSE")},
+ {{std::string("ıi")}, std::string("II")},
{{std::string("àçac123")}, std::string("ÀÇAC123")},
{{std::string("ffi")}, std::string("FFI")},
{{std::string("Dž")}, std::string("DŽ")},
@@ -609,6 +616,59 @@ TEST(function_string_test, function_string_upper_test) {
}
}
+// LOWER/UPPER/INITCAP case-map through the ICU root locale, independent of the
+// process default locale. Under a Turkish (tr_TR) default locale ICU maps
ASCII
+// 'I' -> 'ı' (U+0131) and 'i' -> 'İ' (U+0130); the root locale keeps the plain
+// ASCII round-trip. This test installs a tr_TR ICU default and asserts every
+// result stays locale independent.
+//
+// TransferImpl/InitcapImpl only take the ICU path when the whole input buffer
+// contains a non-ASCII byte; pure-ASCII columns use the locale-neutral SIMD/C
+// fast path. Each data set therefore keeps a non-ASCII anchor row so that
+// check_function_all_arg_comb's batched (non-const) combination routes the
+// ASCII rows through ICU, where the default locale would otherwise leak in.
+TEST(function_string_test, function_string_case_root_locale_test) {
+ UErrorCode status = U_ZERO_ERROR;
+ const icu::Locale saved_default = icu::Locale::getDefault();
+ icu::Locale::setDefault(icu::Locale("tr", "TR"), status);
+ ASSERT_TRUE(U_SUCCESS(status)) << "failed to install tr_TR ICU default
locale";
+ // RAII restore of the process-wide ICU default locale, even if an
assertion
+ // below throws during stack unwinding.
+ Defer restore_default {[&]() {
+ UErrorCode restore_status = U_ZERO_ERROR;
+ icu::Locale::setDefault(saved_default, restore_status);
+ }};
+
+ InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
+ {
+ DataSet data_set = {
+ {{std::string("I")}, std::string("i")}, // tr
default -> "ı"
+ {{std::string("KIZILAY")}, std::string("kizilay")}, // tr
default -> "kızılay"
+ {{std::string("ÀÇ")}, std::string("àç")}, //
non-ASCII anchor: forces ICU
+ };
+ check_function_all_arg_comb<DataTypeString,
true>(std::string("lower"), input_types,
+ data_set);
+ }
+ {
+ DataSet data_set = {
+ {{std::string("i")}, std::string("I")}, // tr
default -> "İ"
+ {{std::string("istanbul")}, std::string("ISTANBUL")}, // tr
default -> "İSTANBUL"
+ {{std::string("straße")}, std::string("STRASSE")}, //
length-changing anchor
+ };
+ check_function_all_arg_comb<DataTypeString,
true>(std::string("upper"), input_types,
+ data_set);
+ }
+ {
+ DataSet data_set = {
+ {{std::string("BIT")}, std::string("Bit")}, //
tr default -> "Bıt"
+ {{std::string("KIZILAY BIT")}, std::string("Kizilay Bit")}, //
tr -> "Kızılay Bıt"
+ {{std::string("ÀÇ")}, std::string("Àç")}, //
non-ASCII anchor
+ };
+ check_function_all_arg_comb<DataTypeString,
true>(std::string("initcap"), input_types,
+ data_set);
+ }
+}
+
TEST(function_string_test, function_string_trim_test) {
std::string func_name = "trim";
{
@@ -3425,7 +3485,8 @@ TEST(function_string_test, function_initcap) {
std::string("Grosse Àstanbul , Àçac123
Σοφος")},
{{std::string("HELLO, WORLD!")}, std::string("Hello,
World!")},
{{std::string("HHHH+-1; asAAss__!")},
std::string("Hhhh+-1; Asaass__!")},
- {{std::string("a,B,C,D")}, std::string("A,B,C,D")}};
+ {{std::string("a,B,C,D")}, std::string("A,B,C,D")},
+ {{std::string("straße")}, std::string("Straße")}};
check_function_all_arg_comb<DataTypeString, true>(func_name, input_types,
data_set);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
