github-actions[bot] commented on code in PR #68484:
URL: https://github.com/apache/doris/pull/68484#discussion_r4095027339
##########
be/src/util/jsonb_parser_simd.h:
##########
@@ -244,38 +233,121 @@ struct JsonbParser {
return Status::OK();
}
- static Status write_number(simdjson::ondemand::number num,
- simdjson ::ondemand::number_type num_type,
+ // Matches the JSON number grammar exactly:
+ // -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
+ static bool is_json_number(std::string_view token) {
+ size_t i = 0;
+ const size_t n = token.size();
+ auto skip_digits = [&]() {
+ const size_t start = i;
+ while (i < n && token[i] >= '0' && token[i] <= '9') {
+ ++i;
+ }
+ return i > start;
+ };
+ if (i < n && token[i] == '-') {
+ ++i;
+ }
+ if (i < n && token[i] == '0') {
+ ++i;
+ } else if (!skip_digits()) {
+ return false;
+ }
+ if (i < n && token[i] == '.') {
+ ++i;
+ if (!skip_digits()) {
+ return false;
+ }
+ }
+ if (i < n && (token[i] == 'e' || token[i] == 'E')) {
+ ++i;
+ if (i < n && (token[i] == '+' || token[i] == '-')) {
+ ++i;
+ }
+ if (!skip_digits()) {
+ return false;
+ }
+ }
+ return i == n;
+ }
+
+ // According to https://github.com/simdjson/simdjson/pull/2139, integers
that do not fit
+ // in 64 bits can be handled by parsing the raw_json_token ourselves:
simdjson returns
+ // NUMBER_ERROR for 18446744073709551616 (one above uint64 max) and
BIGINT_ERROR for
+ // longer integers such as 18446744073709551616231231.
+ // However NUMBER_ERROR is also what simdjson returns for malformed tokens
(leading
+ // zeros like 01, a trailing dot like 1., 1e, trailing garbage like 1x)
and for values
+ // beyond the double range. `num` carries nothing usable in any of these
cases, so the
+ // raw token is first checked against the JSON number grammar and then
parsed as int128
+ // or double.
+ static Status write_number_from_token(simdjson::error_code res,
std::string_view raw_string,
+ JsonbWriter& writer) {
+ // raw_json_token() spans up to the start of the next token, so it may
end with
+ // JSON whitespace.
+ std::string_view token = raw_string;
+ while (!token.empty() && (token.back() == ' ' || token.back() == '\t'
||
+ token.back() == '\n' || token.back() ==
'\r')) {
+ token.remove_suffix(1);
+ }
+ if (!is_json_number(token)) {
+ return Status::InvalidArgument("simdjson get_number failed: {},
raw string is: {}",
Review Comment:
[P2] Bound the raw number included in this error. This branch eagerly
formats the entire token into a `Status`, but `json_valid` and the
error-to-null/value variants immediately discard that text. A malformed row
consisting of `1.`, a multi-megabyte digit run, and `x` therefore creates
another input-sized allocation on top of the source column and simdjson's
padded copy, and can exhaust query memory while merely validating bad data.
Please report a short prefix plus the full token length, and apply the same
bound to the new root trailing-content diagnostic, instead of copying the
unbounded view.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
