airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4065253318
##########
be/src/storage/index/inverted/char_filter/icu_normalizer_char_filter.cpp:
##########
@@ -59,33 +63,97 @@ void ICUNormalizerCharFilter::fill() {
input.resize(_reader->size());
_reader->readCopy(input.data(), 0, static_cast<int32_t>(input.size()));
normalize_text(input, _buf);
+ build_source_byte_offset_runs();
_transformed_input.init(_buf.data(), static_cast<int32_t>(_buf.size()),
false);
}
void ICUNormalizerCharFilter::normalize_text(const std::string& input,
std::string& output) {
output.clear();
+ _edits.reset();
if (input.empty()) {
return;
}
UErrorCode status = U_ZERO_ERROR;
- icu::UnicodeString src16 = icu::UnicodeString::fromUTF8(input);
- UNormalizationCheckResult quick_result = _normalizer->quickCheck(src16,
status);
- if (U_SUCCESS(status) && quick_result == UNORM_YES) {
+ icu::StringByteSink<std::string> sink(&output);
+ _normalizer->normalizeUTF8(0, icu::StringPiece(input), sink, &_edits,
status);
+ if (U_FAILURE(status)) {
+ LOG(WARNING) << "ICU normalize failed: " << u_errorName(status) << ",
using original text";
output = input;
+ _edits.reset();
+ _edits.addUnchanged(static_cast<int32_t>(input.size()));
return;
}
+}
- icu::UnicodeString result16;
- status = U_ZERO_ERROR;
- _normalizer->normalize(src16, result16, status);
+void ICUNormalizerCharFilter::build_source_byte_offset_runs() {
+ _offset_correction_runs.clear();
+ UErrorCode status = U_ZERO_ERROR;
+ auto iterator = _edits.getFineChangesIterator();
+ while (iterator.next(status)) {
+ if (U_FAILURE(status)) {
+ _offset_correction_runs.clear();
+ return;
+ }
+
+ const int32_t source_start = iterator.sourceIndex();
+ const int32_t destination_start = iterator.destinationIndex();
+ const int32_t source_length = iterator.oldLength();
+ const int32_t destination_length = iterator.newLength();
+ if (!_offset_correction_runs.empty()) {
+ auto& previous = _offset_correction_runs.back();
+ const int64_t previous_source_end =
+ static_cast<int64_t>(previous.source_start) +
+ static_cast<int64_t>(previous.source_length) *
previous.repeat_count;
+ const int64_t previous_destination_end =
+ static_cast<int64_t>(previous.destination_start) +
+ static_cast<int64_t>(previous.destination_length) *
previous.repeat_count;
+ if (previous.source_length == source_length &&
+ previous.destination_length == destination_length &&
+ previous_source_end == source_start &&
+ previous_destination_end == destination_start) {
+ ++previous.repeat_count;
+ continue;
+ }
+ }
+ _offset_correction_runs.push_back(
+ {source_start, destination_start, source_length,
destination_length, 1});
+ }
if (U_FAILURE(status)) {
- LOG(WARNING) << "ICU normalize failed: " << u_errorName(status) << ",
using original text";
- output = input;
- return;
+ _offset_correction_runs.clear();
}
+}
- result16.toUTF8String(output);
+int32_t ICUNormalizerCharFilter::correct_offset(int32_t current_offset) const {
+ if (current_offset < 0 || _offset_correction_runs.empty()) {
+ return DorisCharFilter::correct_offset(current_offset);
+ }
+
+ const auto next_run = std::ranges::upper_bound(_offset_correction_runs,
current_offset, {},
+
&OffsetCorrectionRun::destination_start);
+ if (next_run == _offset_correction_runs.begin()) {
+ return DorisCharFilter::correct_offset(current_offset);
+ }
+
+ const auto& run = *std::prev(next_run);
+ const int64_t source_end = static_cast<int64_t>(run.source_start) +
+ static_cast<int64_t>(run.source_length) *
run.repeat_count;
+ const int64_t destination_end =
static_cast<int64_t>(run.destination_start) +
+
static_cast<int64_t>(run.destination_length) * run.repeat_count;
+ if (run.destination_length == 0 && current_offset ==
run.destination_start) {
+ return
DorisCharFilter::correct_offset(static_cast<int32_t>(source_end));
+ }
+
+ if (current_offset <= destination_end) {
Review Comment:
Fixed in 31450230. Tokenizer provenance now converts repeated corrected
boundaries into conservative per-rune start/end intervals. The
ligature-expansion test verifies both normalized `f` and `i` retain source span
`[0,3)` through offset-aware Pinyin, followed by correct reset behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
