amorynan commented on code in PR #20556:
URL: https://github.com/apache/doris/pull/20556#discussion_r1226021428
##########
be/src/vec/data_types/data_type_array.cpp:
##########
@@ -267,56 +239,16 @@ Status DataTypeArray::from_string(ReadBuffer& rb,
IColumn* column) const {
return Status::InvalidArgument("Array does not end with ']' character,
found '{}'",
*(rb.end() - 1));
}
- // empty array []
- if (rb.count() == 2) {
- offsets.push_back(offsets.back());
- return Status::OK();
- }
- ++rb.position();
-
- size_t element_num = 0;
- // parse array element until end of array
- while (!rb.eof()) {
- StringRef element(rb.position(), rb.count());
- bool has_quota = false;
- if (!next_element_from_string(rb, element, has_quota)) {
- // we should do array element column revert if error
- nested_column.pop_back(element_num);
- return Status::InvalidArgument("Cannot read array element from
text '{}'",
- element.to_string());
- }
-
- // handle empty element
- if (element.size == 0) {
- auto& nested_null_col =
reinterpret_cast<ColumnNullable&>(nested_column);
- nested_null_col.get_nested_column().insert_default();
- nested_null_col.get_null_map_data().push_back(0);
- ++element_num;
- continue;
- }
-
- // handle null element, need to distinguish null and "null"
- if (!has_quota && element.size == 4 && strncmp(element.data, "null",
4) == 0) {
- // insert null
- auto& nested_null_col =
reinterpret_cast<ColumnNullable&>(nested_column);
- nested_null_col.get_nested_column().insert_default();
- nested_null_col.get_null_map_data().push_back(1);
- ++element_num;
- continue;
- }
-
- // handle normal element
- ReadBuffer read_buffer(const_cast<char*>(element.data), element.size);
- auto st = nested->from_string(read_buffer, &nested_column);
- if (!st.ok()) {
- // we should do array element column revert if error
- nested_column.pop_back(element_num);
- return st;
- }
- ++element_num;
- }
- offsets.push_back(offsets.back() + element_num);
- return Status::OK();
+ // json parser
+ std::unique_ptr<simdjson::ondemand::parser> _ondemand_json_parser =
+ std::make_unique<simdjson::ondemand::parser>();
+ size_t _padded_size = rb.count() + simdjson::SIMDJSON_PADDING;
+ std::string _simdjson_ondemand_padding_buffer;
+ _simdjson_ondemand_padding_buffer.resize(_padded_size);
+ memcpy(&_simdjson_ondemand_padding_buffer.front(), rb.position(),
rb.count());
+ simdjson::ondemand::document array_doc = _ondemand_json_parser->iterate(
+ std::string_view(_simdjson_ondemand_padding_buffer.data(),
rb.count()), _padded_size);
+ auto value = array_doc.get_value();
Review Comment:
I think it should in from_json begin avoid other complex type to call
from_json
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
