freemandealer commented on code in PR #27272:
URL: https://github.com/apache/doris/pull/27272#discussion_r1403402330
##########
be/src/vec/exec/format/json/new_json_reader.cpp:
##########
@@ -1437,48 +1459,38 @@ Status
NewJsonReader::_append_error_msg(simdjson::ondemand::object* obj, std::st
return Status::OK();
}
-Status NewJsonReader::_simdjson_parse_json(bool* is_empty_row, bool* eof) {
- size_t size = 0;
- RETURN_IF_ERROR(_simdjson_parse_json_doc(&size, eof));
-
- // read all data, then return
- if (size == 0 || *eof) {
- *is_empty_row = true;
+Status NewJsonReader::_simdjson_parse_json(size_t* size, bool* is_empty_row,
bool* eof,
+ simdjson::error_code* error) {
+ SCOPED_TIMER(_file_read_timer);
+ // step1: read buf from pipe.
+ RETURN_IF_ERROR(_read_pipe_buf(size, eof));
+ if (*eof) {
return Status::OK();
}
- if (!_parsed_jsonpaths.empty() && _strip_outer_array) {
- _total_rows = _json_value.count_elements().value();
- _next_row = 0;
-
- if (_total_rows == 0) {
- // meet an empty json array.
- *is_empty_row = true;
- }
- }
+ // step2: init json parser iterate.
+ RETURN_IF_ERROR(_init_json_parser_iterate(size, error));
return Status::OK();
}
-Status NewJsonReader::_simdjson_parse_json_doc(size_t* size, bool* eof) {
- // read a whole message
- SCOPED_TIMER(_file_read_timer);
- const uint8_t* json_str = nullptr;
- std::unique_ptr<uint8_t[]> json_str_ptr;
+
+Status NewJsonReader::_read_pipe_buf(size_t* size, bool* eof) {
Review Comment:
the function name `read_pipe_buf` is not suitable here for the data source
is not necessarily a pipe.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
