This is an automated email from the ASF dual-hosted git repository. eldenmoon pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new 70c308c5baf [Pick](Variant) pick some fix (#38513) 70c308c5baf is described below commit 70c308c5bafbcffab6185fbf86ddd50f5850bcfd Author: lihangyu <15605149...@163.com> AuthorDate: Tue Jul 30 13:42:51 2024 +0800 [Pick](Variant) pick some fix (#38513) #38318 #38364 #38291 #38213 #38413 --- be/src/agent/be_exec_version_manager.h | 4 +- be/src/exprs/json_functions.cpp | 5 + be/src/exprs/json_functions.h | 2 + be/src/olap/rowset/segment_v2/segment_iterator.cpp | 5 + be/src/vec/columns/column_object.cpp | 61 +++++++++- be/src/vec/columns/column_object.h | 2 + be/src/vec/data_types/data_type_object.cpp | 18 ++- be/src/vec/exec/format/json/new_json_reader.cpp | 14 ++- be/src/vec/functions/function_cast.h | 17 ++- be/src/vec/functions/function_variant_element.cpp | 10 +- .../main/java/org/apache/doris/common/Config.java | 2 +- .../data/load_p0/stream_load/test_json_load.out | 6 + .../load_p0/stream_load/test_read_root_path.json | 4 + regression-test/data/variant_p0/rqg/rqg2.out | 48 ++++++++ regression-test/data/variant_p0/rqg/rqg3.out | 130 +++++++++++++++++++++ regression-test/data/variant_p0/rqg/rqg4.out | 16 +++ regression-test/data/variant_p0/rqg/rqg5.out | 67 +++++++++++ .../load_p0/stream_load/test_json_load.groovy | 27 +++++ .../nereids_rules_p0/mv/variant/variant_mv.groovy | 2 +- regression-test/suites/variant_p0/rqg/rqg2.sql | 8 ++ regression-test/suites/variant_p0/rqg/rqg3.sql | 43 +++++++ regression-test/suites/variant_p0/rqg/rqg4.sql | 6 + regression-test/suites/variant_p0/rqg/rqg5.sql | 22 ++++ 23 files changed, 508 insertions(+), 11 deletions(-) diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h index ec6ddf497ec..a55e26f7ba4 100644 --- a/be/src/agent/be_exec_version_manager.h +++ b/be/src/agent/be_exec_version_manager.h @@ -80,8 +80,9 @@ private: * b. clear old version of version 3->4 * c. change FunctionIsIPAddressInRange from AlwaysNotNullable to DependOnArguments * d. change some agg function nullable property: PR #37215 + * e. change variant serde to fix PR #38413 */ -constexpr inline int BeExecVersionManager::max_be_exec_version = 5; +constexpr inline int BeExecVersionManager::max_be_exec_version = 6; constexpr inline int BeExecVersionManager::min_be_exec_version = 0; /// functional @@ -89,5 +90,6 @@ constexpr inline int BITMAP_SERDE = 3; constexpr inline int USE_NEW_SERDE = 4; // release on DORIS version 2.1 constexpr inline int OLD_WAL_SERDE = 3; // use to solve compatibility issues, see pr #32299 constexpr inline int AGG_FUNCTION_NULLABLE = 5; // change some agg nullable property: PR #37215 +constexpr inline int VARIANT_SERDE = 6; // change variant serde to fix PR #38413 } // namespace doris diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp index 205ee5a5d20..5e3fb136929 100644 --- a/be/src/exprs/json_functions.cpp +++ b/be/src/exprs/json_functions.cpp @@ -353,4 +353,9 @@ void JsonFunctions::merge_objects(rapidjson::Value& dst_object, rapidjson::Value } } +// root path "$." +bool JsonFunctions::is_root_path(const std::vector<JsonPath>& json_path) { + return json_path.size() == 2 && json_path[0].key == "$" && json_path[1].key.empty(); +} + } // namespace doris diff --git a/be/src/exprs/json_functions.h b/be/src/exprs/json_functions.h index 72aa522ff37..11970eb8c46 100644 --- a/be/src/exprs/json_functions.h +++ b/be/src/exprs/json_functions.h @@ -116,6 +116,8 @@ public: static std::string print_json_value(const rapidjson::Value& value); + static bool is_root_path(const std::vector<JsonPath>& json_path); + private: static rapidjson::Value* match_value(const std::vector<JsonPath>& parsed_paths, rapidjson::Value* document, diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index f44e32f4687..2d0a85e1750 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -697,6 +697,11 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row if (_opts.io_ctx.reader_type == ReaderType::READER_QUERY) { RowRanges dict_row_ranges = RowRanges::create_single(num_rows()); for (auto cid : cids) { + if (!_segment->can_apply_predicate_safely(cid, + _opts.col_id_to_predicates.at(cid).get(), + *_schema, _opts.io_ctx.reader_type)) { + continue; + } RowRanges tmp_row_ranges = RowRanges::create_single(num_rows()); DCHECK(_opts.col_id_to_predicates.count(cid) > 0); RETURN_IF_ERROR(_column_iterators[cid]->get_row_ranges_by_dict( diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index b56c4857334..65c0a5dcd89 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -509,9 +509,24 @@ MutableColumnPtr ColumnObject::apply_for_subcolumns(Func&& func) const { res->add_sub_column(subcolumn->path, new_subcolumn->assume_mutable(), subcolumn->data.get_least_common_type()); } + check_consistency(); return res; } +void ColumnObject::resize(size_t n) { + if (n == num_rows) { + return; + } + if (n > num_rows) { + insert_many_defaults(n - num_rows); + } else { + for (auto& subcolumn : subcolumns) { + subcolumn->data.pop_back(num_rows - n); + } + } + num_rows = n; +} + bool ColumnObject::Subcolumn::check_if_sparse_column(size_t num_rows) { if (num_rows < config::variant_threshold_rows_to_estimate_sparse_column) { return false; @@ -697,8 +712,16 @@ MutableColumnPtr ColumnObject::clone_resized(size_t new_size) const { if (new_size == 0) { return ColumnObject::create(is_nullable); } - return apply_for_subcolumns( + // If subcolumns are empty, then res will be empty but new_size > 0 + if (subcolumns.empty()) { + // Add an emtpy column with new_size rows + auto res = ColumnObject::create(true, false); + res->set_num_rows(new_size); + return res; + } + auto res = apply_for_subcolumns( [&](const auto& subcolumn) { return subcolumn.clone_resized(new_size); }); + return res; } size_t ColumnObject::byte_size() const { @@ -838,7 +861,10 @@ Field ColumnObject::operator[](size_t n) const { } void ColumnObject::get(size_t n, Field& res) const { - assert(n < size()); + if (UNLIKELY(n >= size())) { + throw doris::Exception(ErrorCode::OUT_OF_BOUND, + "Index ({}) for getting field is out of range", n); + } res = VariantMap(); auto& object = res.get<VariantMap&>(); @@ -886,11 +912,32 @@ void ColumnObject::insert_range_from(const IColumn& src, size_t start, size_t le } ColumnPtr ColumnObject::replicate(const Offsets& offsets) const { + if (subcolumns.empty()) { + // Add an emtpy column with offsets.back rows + auto res = ColumnObject::create(true, false); + res->set_num_rows(offsets.back()); + } return apply_for_subcolumns( [&](const auto& subcolumn) { return subcolumn.replicate(offsets); }); } ColumnPtr ColumnObject::permute(const Permutation& perm, size_t limit) const { + if (subcolumns.empty()) { + if (limit == 0) { + limit = num_rows; + } else { + limit = std::min(num_rows, limit); + } + + if (perm.size() < limit) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "Size of permutation is less than required."); + } + // Add an emtpy column with limit rows + auto res = ColumnObject::create(true, false); + res->set_num_rows(limit); + return res; + } return apply_for_subcolumns( [&](const auto& subcolumn) { return subcolumn.permute(perm, limit); }); } @@ -1420,6 +1467,12 @@ ColumnPtr ColumnObject::filter(const Filter& filter, ssize_t count) const { return finalized_object.apply_for_subcolumns( [&](const auto& subcolumn) { return subcolumn.filter(filter, count); }); } + if (subcolumns.empty()) { + // Add an emtpy column with filtered rows + auto res = ColumnObject::create(true, false); + res->set_num_rows(count_bytes_in_filter(filter)); + return res; + } auto new_column = ColumnObject::create(true, false); for (auto& entry : subcolumns) { auto subcolumn = entry->data.get_finalized_column().filter(filter, count); @@ -1433,6 +1486,10 @@ Status ColumnObject::filter_by_selector(const uint16_t* sel, size_t sel_size, IC if (!is_finalized()) { finalize(); } + if (subcolumns.empty()) { + assert_cast<ColumnObject*>(col_ptr)->insert_many_defaults(sel_size); + return Status::OK(); + } auto* res = assert_cast<ColumnObject*>(col_ptr); for (const auto& subcolumn : subcolumns) { auto new_subcolumn = subcolumn->data.get_least_common_type()->create_column(); diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h index 441589bdfbb..e9b6eb7dfd8 100644 --- a/be/src/vec/columns/column_object.h +++ b/be/src/vec/columns/column_object.h @@ -363,6 +363,8 @@ public: void clear() override; + void resize(size_t n) override; + void clear_subcolumns_data(); std::string get_name() const override { diff --git a/be/src/vec/data_types/data_type_object.cpp b/be/src/vec/data_types/data_type_object.cpp index 7a75583cd7b..c3c43c1bf69 100644 --- a/be/src/vec/data_types/data_type_object.cpp +++ b/be/src/vec/data_types/data_type_object.cpp @@ -29,6 +29,7 @@ #include <utility> #include <vector> +#include "agent/be_exec_version_manager.h" #include "vec/columns/column_object.h" #include "vec/common/assert_cast.h" #include "vec/common/typeid_cast.h" @@ -66,7 +67,6 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column, if (is_nothing(type)) { continue; } - PColumnMeta column_meta_pb; column_meta_pb.set_name(entry->path.get_path()); type->to_pb_column_meta(&column_meta_pb); @@ -78,6 +78,10 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column, size += type->get_uncompressed_serialized_bytes(entry->data.get_finalized_column(), be_exec_version); } + // serialize num of rows, only take effect when subcolumns empty + if (be_exec_version >= VARIANT_SERDE) { + size += sizeof(uint32_t); + } return size; } @@ -121,6 +125,11 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve } // serialize num of subcolumns *reinterpret_cast<uint32_t*>(size_pos) = num_of_columns; + // serialize num of rows, only take effect when subcolumns empty + if (be_exec_version >= VARIANT_SERDE) { + *reinterpret_cast<uint32_t*>(buf) = column_object.rows(); + buf += sizeof(uint32_t); + } return buf; } @@ -155,6 +164,13 @@ const char* DataTypeObject::deserialize(const char* buf, IColumn* column, } column_object->add_sub_column(key, std::move(sub_column), type); } + size_t num_rows = 0; + // serialize num of rows, only take effect when subcolumns empty + if (be_exec_version >= VARIANT_SERDE) { + num_rows = *reinterpret_cast<const uint32_t*>(buf); + column_object->set_num_rows(num_rows); + buf += sizeof(uint32_t); + } column_object->finalize(); #ifndef NDEBUG diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 7d97aabe39c..f3dab071e73 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -1656,7 +1656,19 @@ Status NewJsonReader::_simdjson_write_columns_by_jsonpath( return st; } } - if (i >= _parsed_jsonpaths.size() || st.is<NOT_FOUND>()) { + if (i < _parsed_jsonpaths.size() && JsonFunctions::is_root_path(_parsed_jsonpaths[i])) { + // Indicate that the jsonpath is "$.", read the full root json object, insert the original doc directly + ColumnNullable* nullable_column = nullptr; + IColumn* target_column_ptr = nullptr; + if (slot_desc->is_nullable()) { + nullable_column = assert_cast<ColumnNullable*>(column_ptr); + target_column_ptr = &nullable_column->get_nested_column(); + } + auto* column_string = assert_cast<ColumnString*>(target_column_ptr); + column_string->insert_data(_simdjson_ondemand_padding_buffer.data(), + _original_doc_size); + has_valid_value = true; + } else if (i >= _parsed_jsonpaths.size() || st.is<NOT_FOUND>()) { // not match in jsondata, filling with default value RETURN_IF_ERROR(_fill_missing_column(slot_desc, column_ptr, valid)); if (!(*valid)) { diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 563eac408d2..af2fadc84c2 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -719,14 +719,25 @@ struct ConvertImplGenericFromJsonb { } // add string to string column if (context->jsonb_string_as_string() && is_dst_string && value->isString()) { - auto blob = static_cast<const JsonbBlobVal*>(value); + const auto* blob = static_cast<const JsonbBlobVal*>(value); assert_cast<ColumnString&>(*col_to).insert_data(blob->getBlob(), blob->getBlobLen()); (*vec_null_map_to)[i] = 0; continue; } - std::string json_str = JsonbToJson::jsonb_to_json_string(val.data, val.size); - ReadBuffer read_buffer((char*)(json_str.data()), json_str.size()); + std::string input_str; + if (context->jsonb_string_as_string() && value->isString()) { + const auto* blob = static_cast<const JsonbBlobVal*>(value); + input_str = std::string(blob->getBlob(), blob->getBlobLen()); + } else { + input_str = JsonbToJson::jsonb_to_json_string(val.data, val.size); + } + if (input_str.empty()) { + col_to->insert_default(); + (*vec_null_map_to)[i] = 1; + continue; + } + ReadBuffer read_buffer((char*)(input_str.data()), input_str.size()); Status st = data_type_to->from_string(read_buffer, col_to); // if parsing failed, will return null (*vec_null_map_to)[i] = !st.ok(); diff --git a/be/src/vec/functions/function_variant_element.cpp b/be/src/vec/functions/function_variant_element.cpp index 84ddc3b8046..faa56ccb1e3 100644 --- a/be/src/vec/functions/function_variant_element.cpp +++ b/be/src/vec/functions/function_variant_element.cpp @@ -28,6 +28,7 @@ #include "common/status.h" #include "exprs/json_functions.h" #include "simdjson.h" +#include "util/defer_op.h" #include "vec/columns/column.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_object.h" @@ -102,8 +103,11 @@ private: static Status get_element_column(const ColumnObject& src, const ColumnPtr& index_column, ColumnPtr* result) { std::string field_name = index_column->get_data_at(0).to_string(); + Defer finalize([&]() { (*result)->assume_mutable()->finalize(); }); if (src.empty()) { *result = ColumnObject::create(true); + // src subcolumns empty but src row count may not be 0 + (*result)->assume_mutable()->insert_many_defaults(src.size()); return Status::OK(); } if (src.is_scalar_variant() && @@ -135,8 +139,10 @@ private: PathInData path(field_name); ColumnObject::Subcolumns subcolumns = mutable_ptr->get_subcolumns(); const auto* node = subcolumns.find_exact(path); - auto result_col = ColumnObject::create(true, false /*should not create root*/); + MutableColumnPtr result_col; if (node != nullptr) { + // Create without root, since root will be added + result_col = ColumnObject::create(true, false /*should not create root*/); std::vector<decltype(node)> nodes; PathsInData paths; ColumnObject::Subcolumns::get_leaves_of_node(node, nodes, paths); @@ -162,6 +168,8 @@ private: auto container = ColumnObject::create(std::move(new_subcolumns), true); result_col->insert_range_from(*container, 0, container->size()); } else { + // Create with root, otherwise the root type maybe type Nothing + result_col = ColumnObject::create(true); result_col->insert_many_defaults(src.size()); } *result = result_col->get_ptr(); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 7af0da4f9d6..55ef9961e50 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1825,7 +1825,7 @@ public class Config extends ConfigBase { * Max data version of backends serialize block. */ @ConfField(mutable = false) - public static int max_be_exec_version = 5; + public static int max_be_exec_version = 6; /** * Min data version of backends serialize block. diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out index 588b6edb004..7df15b74b86 100644 --- a/regression-test/data/load_p0/stream_load/test_json_load.out +++ b/regression-test/data/load_p0/stream_load/test_json_load.out @@ -250,3 +250,9 @@ test k2_value -- !select29 -- 10 \N + +-- !select30 -- +12345 {"k1":12345,"k2":"11111","k3":111111,"k4":[11111]} {"k1":12345,"k2":"11111","k3":111111,"k4":[11111]} 111111 +12346 {"k1":12346,"k2":"22222","k4":[22222]} {"k1":12346,"k2":"22222","k4":[22222]} \N +12347 {"k1":12347,"k3":"33333","k4":[22222]} {"k1":12347,"k3":"33333","k4":[22222]} 33333 +12348 {"k1":12348,"k3":"33333","k5":{"k51":1024,"xxxx":[11111]}} {"k1":12348,"k3":"33333","k5":{"k51":1024,"xxxx":[11111]}} 33333 \ No newline at end of file diff --git a/regression-test/data/load_p0/stream_load/test_read_root_path.json b/regression-test/data/load_p0/stream_load/test_read_root_path.json new file mode 100644 index 00000000000..777ccbbfb1f --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_read_root_path.json @@ -0,0 +1,4 @@ +{"k1" : 12345, "k2" : "11111", "k3" : 111111, "k4" : [11111]} +{"k1" : 12346, "k2" : "22222", "k4" : [22222]} +{"k1" : 12347, "k3" : "33333", "k4" : [22222]} +{"k1" : 12348, "k3" : "33333", "k5" : {"k51" : 1024, "xxxx" : [11111]}} \ No newline at end of file diff --git a/regression-test/data/variant_p0/rqg/rqg2.out b/regression-test/data/variant_p0/rqg/rqg2.out new file mode 100644 index 00000000000..7c006a64aec --- /dev/null +++ b/regression-test/data/variant_p0/rqg/rqg2.out @@ -0,0 +1,48 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rqg2 -- +0 + +-- !rqg2_2 -- +500 + +-- !rqg2_3 -- +500 + +-- !rqg2_4 -- +0 + +-- !rqg2_5 -- +0 + +-- !rqg2_6 -- +500 + +-- !rqg2_7 -- +500 + +-- !rqg2_8 -- +2023-12-09 +2023-12-10 +2023-12-11 +2023-12-12 +2023-12-13 +2023-12-14 +2023-12-15 +2023-12-16 +2023-12-17 +2023-12-18 +2023-12-19 +2023-12-20 +2024-01-08 +2024-01-09 +2024-01-17 +2024-01-19 +2024-01-31 +2024-02-18 +2025-02-18 +2025-06-18 +2026-01-18 +2026-02-18 +2027-01-09 +2027-01-16 + diff --git a/regression-test/data/variant_p0/rqg/rqg3.out b/regression-test/data/variant_p0/rqg/rqg3.out new file mode 100644 index 00000000000..d39b0e288dc --- /dev/null +++ b/regression-test/data/variant_p0/rqg/rqg3.out @@ -0,0 +1,130 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rqg3 -- +0 + +-- !rqg3_2 -- +0 + +-- !rqg3_3 -- +0 + +-- !rqg3_4 -- +0 + +-- !rqg3_5 -- +0 + +-- !rqg3_6 -- +0 + +-- !rqg3_7 -- +0 + +-- !rqg3_8 -- +0 + +-- !rqg3_9 -- +0 + +-- !rqg3_10 -- +0 + +-- !rqg3_11 -- +10 + +-- !rqg3_12 -- +10 + +-- !rqg3_13 -- +10 + +-- !rqg3_14 -- +10 + +-- !rqg3_15 -- +10 + +-- !rqg3_16 -- +10 + +-- !rqg3_17 -- +10 + +-- !rqg3_18 -- +10 + +-- !rqg3_19 -- +21 + +-- !rqg3_20 -- +21 + +-- !rqg3_21 -- +21 + +-- !rqg3_22 -- +21 + +-- !rqg3_23 -- +21 + +-- !rqg3_24 -- +21 + +-- !rqg3_25 -- +25 + +-- !rqg3_26 -- +25 + +-- !rqg3_27 -- +25 + +-- !rqg3_28 -- +25 + +-- !rqg3_29 -- +25 + +-- !rqg3_30 -- +25 + +-- !rqg3_31 -- +6 + +-- !rqg3_32 -- +6 + +-- !rqg3_33 -- +6 + +-- !rqg3_34 -- +6 + +-- !rqg3_35 -- +6 + +-- !rqg3_36 -- +6 + +-- !rqg3_37 -- +7 + +-- !rqg3_38 -- +7 + +-- !rqg3_39 -- +7 + +-- !rqg3_40 -- +7 + +-- !rqg3_41 -- +7 + +-- !rqg3_42 -- +7 + +-- !rqg3_43 -- +0 \N c \N \N \N + diff --git a/regression-test/data/variant_p0/rqg/rqg4.out b/regression-test/data/variant_p0/rqg/rqg4.out new file mode 100644 index 00000000000..ec975af29e3 --- /dev/null +++ b/regression-test/data/variant_p0/rqg/rqg4.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rqg4 -- +0 + +-- !rqg4_2 -- +500 + +-- !rqg4_3 -- +500 + +-- !rqg4_4 -- +70 + +-- !rqg4_5 -- +70 + diff --git a/regression-test/data/variant_p0/rqg/rqg5.out b/regression-test/data/variant_p0/rqg/rqg5.out new file mode 100644 index 00000000000..c51e79dab4b --- /dev/null +++ b/regression-test/data/variant_p0/rqg/rqg5.out @@ -0,0 +1,67 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rqg5 -- +0 + +-- !rqg5_2 -- +50 + +-- !rqg5_3 -- +50 + +-- !rqg5_4 -- +50 + +-- !rqg5_5 -- +50 + +-- !rqg5_6 -- +50 + +-- !rqg5_7 -- +0 + +-- !rqg5_8 -- +0 + +-- !rqg5_9 -- +50 + +-- !rqg5_10 -- +50 + +-- !rqg5_11 -- +50 + +-- !rqg5_12 -- +50 + +-- !rqg5_13 -- +50 + +-- !rqg5_14 -- +50 + +-- !rqg5_15 -- +0 + +-- !rqg5_16 -- +100 + +-- !rqg5_17 -- +100 + +-- !rqg5_18 -- +100 + +-- !rqg5_19 -- +100 + +-- !rqg5_20 -- +100 + +-- !rqg5_21 -- +100 + +-- !rqg5_22 -- +0 + diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index e2235c7d5b3..1cf2108d48e 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -878,4 +878,31 @@ suite("test_json_load", "p0,nonConcurrent") { } finally { try_sql("DROP TABLE IF EXISTS ${testTable}") } + + // support read "$." as root + try { + sql "DROP TABLE IF EXISTS ${testTable}" + sql """CREATE TABLE IF NOT EXISTS ${testTable} + ( + `k1` varchar(1024) NULL, + `k2` variant NULL, + `k3` variant NULL, + `k4` variant NULL + ) + DUPLICATE KEY(`k1`) + COMMENT '' + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + load_json_data.call("${testTable}", "${testTable}_case30", 'false', 'true', 'json', '', '[\"$.k1\",\"$.\", \"$.\", \"$.k3\"]', + '', '', '', 'test_read_root_path.json') + + sql "sync" + qt_select30 "select * from ${testTable} order by k1" + + } finally { + // try_sql("DROP TABLE IF EXISTS ${testTable}") + } } diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy index b1c4d371dbb..23f5e889e78 100644 --- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy @@ -573,7 +573,7 @@ suite("variant_mv") { where g2.actor['id'] > 34259289; """ def query3_6 = """ - SELECT + SELECT /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=0,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=32,parallel_scan_min_rows_per_scanner=64,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=1,enable_parallel_result_sink=false,ena [...] g1.id, g2.type, floor(cast(g1.actor['id'] as int) + 100.5), diff --git a/regression-test/suites/variant_p0/rqg/rqg2.sql b/regression-test/suites/variant_p0/rqg/rqg2.sql new file mode 100644 index 00000000000..a8146c7bce1 --- /dev/null +++ b/regression-test/suites/variant_p0/rqg/rqg2.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS table_500_undef_partitions2_keys3_properties4_distributed_by5 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_500_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 7, \"col_int_undef_signed2\": 1, \"col_date_undef_signed\": \"2023-12-10\", \"col_date_undef_signed2\": \"2023-12-14\", \"col_varchar_10__undef_signed\": \"p\", \"col_varchar_1024__undef_signed\": \"h\"}'),('1','{\"col_int_undef_signed\": 6, \"col_int_undef_signed2\": 7, \"col_date_undef_signed\": \"2024-01-09\", \"col_date_undef_signed2\": \"2024-02-18\", \"col_varch [...] +INSERT INTO table_500_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 6, \"col_int_undef_signed2\": 3, \"col_date_undef_signed\": \"2027-01-09\", \"col_date_undef_signed2\": \"2023-12-18\", \"col_varchar_10__undef_signed\": \"u\", \"col_varchar_1024__undef_signed\": \"s\"}'),('1','{\"col_int_undef_signed\": 1, \"col_int_undef_signed2\": 7, \"col_date_undef_signed\": \"2026-02-18\", \"col_date_undef_signed2\": \"2025-02-18\", \"col_varch [...] +CREATE TABLE IF NOT EXISTS table_500_undef_partitions2_keys3_properties4_distributed_by55 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_500_undef_partitions2_keys3_properties4_distributed_by55 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_500_undef_partitions2_keys3_properties4_distributed_by55(pk,var) VALUES ('0','{\"col_int_undef_signed\": 2, \"col_int_undef_signed2\": null, \"col_date_undef_signed\": \"2023-12-17\", \"col_date_undef_signed2\": \"2024-02-18\", \"col_varchar_10__undef_signed\": null, \"col_varchar_1024__undef_signed\": \"d\"}'),('1','{\"col_int_undef_signed\": 9, \"col_int_undef_signed2\": null, \"col_date_undef_signed\": \"2023-12-10\", \"col_date_undef_signed2\": \"2024-01-08\", \"col [...] +INSERT INTO table_500_undef_partitions2_keys3_properties4_distributed_by55(pk,var) VALUES ('0','{\"col_int_undef_signed\": 1, \"col_int_undef_signed2\": 4, \"col_date_undef_signed\": \"2025-02-18\", \"col_date_undef_signed2\": \"2024-01-19\", \"col_varchar_10__undef_signed\": null, \"col_varchar_1024__undef_signed\": \"k\"}'),('1','{\"col_int_undef_signed\": null, \"col_int_undef_signed2\": 0, \"col_date_undef_signed\": \"2023-12-10\", \"col_date_undef_signed2\": \"2023-12-17\", \"col_va [...] +SELECT CAST(table1 . var['col_date_undef_signed'] AS date) AS field1 FROM table_500_undef_partitions2_keys3_properties4_distributed_by55 AS table1 JOIN table_500_undef_partitions2_keys3_properties4_distributed_by5 AS table2 ON ( CAST(table2 . var['col_date_undef_signed'] AS date) = CAST(table1 . var['col_date_undef_signed'] AS date) ) WHERE ( CAST(table1 . var['col_int_undef_signed'] AS int) IN ( 1 ) ) GROUP BY field1 ORDER BY field1 LIMIT 10000; diff --git a/regression-test/suites/variant_p0/rqg/rqg3.sql b/regression-test/suites/variant_p0/rqg/rqg3.sql new file mode 100644 index 00000000000..d7fde8f1a1c --- /dev/null +++ b/regression-test/suites/variant_p0/rqg/rqg3.sql @@ -0,0 +1,43 @@ +CREATE TABLE IF NOT EXISTS table_10_undef_partitions2_keys3_properties4_distributed_by5 ( `col_int_undef_signed` int, `col_varchar_10__undef_signed` varchar(10), `col_varchar_1024__undef_signed` varchar(1024), `pk` int, var VARIANT NULL ) engine=olap DUPLICATE KEY(`col_int_undef_signed`) distributed by hash(`col_int_undef_signed`) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_10_undef_partitions2_keys3_properties4_distributed_by5 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_21_undef_partitions2_keys3_properties4_distributed_by5 ( `col_int_undef_signed` int, `col_varchar_10__undef_signed` varchar(10), `col_varchar_1024__undef_signed` varchar(1024), `pk` int, var VARIANT NULL ) engine=olap DUPLICATE KEY(`col_int_undef_signed`) distributed by hash(`col_int_undef_signed`) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_21_undef_partitions2_keys3_properties4_distributed_by5 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_25_undef_partitions2_keys3_properties4_distributed_by5 ( `col_int_undef_signed` int, `col_varchar_10__undef_signed` varchar(10), `col_varchar_1024__undef_signed` varchar(1024), `pk` int, var VARIANT NULL ) engine=olap DUPLICATE KEY(`col_int_undef_signed`) distributed by hash(`col_int_undef_signed`) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_25_undef_partitions2_keys3_properties4_distributed_by5 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_6_undef_partitions2_keys3_properties4_distributed_by5 ( `col_int_undef_signed` int, `col_varchar_10__undef_signed` varchar(10), `col_varchar_1024__undef_signed` varchar(1024), `pk` int, var VARIANT NULL ) engine=olap DUPLICATE KEY(`col_int_undef_signed`) distributed by hash(`col_int_undef_signed`) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_6_undef_partitions2_keys3_properties4_distributed_by5 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_7_undef_partitions2_keys3_properties4_distributed_by5 ( `col_int_undef_signed` int, `col_varchar_10__undef_signed` varchar(10), `col_varchar_1024__undef_signed` varchar(1024), `pk` int, var VARIANT NULL ) engine=olap DUPLICATE KEY(`col_int_undef_signed`) distributed by hash(`col_int_undef_signed`) buckets 10 properties("replication_num" = "1"); +CREATE TABLE IF NOT EXISTS table_7_undef_partitions2_keys3_properties4_distributed_by5 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_bigint_undef_signed\": -94, \"col_varchar_10__undef_signed\": \"had\", \"col_varchar_64__undef_signed\": \"y\"}'),('1','{\"col_bigint_undef_signed\": 672609, \"col_varchar_10__undef_signed\": \"k\", \"col_varchar_64__undef_signed\": \"h\"}'),('2','{\"col_bigint_undef_signed\": -3766684, \"col_varchar_10__undef_signed\": \"a\", \"col_varchar_64__undef_signed\": \"p\"}'),('3','{\"col_bigint [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_bigint_undef_signed\": 111, \"col_varchar_10__undef_signed\": \"from\", \"col_varchar_64__undef_signed\": \"t\"}'),('1','{\"col_bigint_undef_signed\": null, \"col_varchar_10__undef_signed\": \"h\", \"col_varchar_64__undef_signed\": \"out\"}'),('2','{\"col_bigint_undef_signed\": 3814, \"col_varchar_10__undef_signed\": \"get\", \"col_varchar_64__undef_signed\": \"q\"}'),('3','{\"col_bigint_ [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 7, \"col_int_undef_signed_not_null\": 9, \"col_varchar_10__undef_signed\": \"\", \"col_varchar_10__undef_signed_not_null\": \"\"}'),('1','{\"col_int_undef_signed\": 8, \"col_int_undef_signed_not_null\": 6, \"col_varchar_10__undef_signed\": null, \"col_varchar_10__undef_signed_not_null\": \"k\"}'),('2','{\"col_int_undef_signed\": 1, \"col_int_undef_signed_not_null\": 6, [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 7, \"col_varchar_10__undef_signed\": \"\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": null}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"\"}'),('3','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"know\"}'),('4','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": null}' [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 9, \"col_int_undef_signed_not_null\": 8, \"col_date_undef_signed\": \"2023-12-16\", \"col_date_undef_signed_not_null\": \"2023-12-12\", \"col_varchar_10__undef_signed\": \"at\", \"col_varchar_10__undef_signed_not_null\": \"one\", \"col_varchar_1024__undef_signed\": \"r\", \"col_varchar_1024__undef_signed_not_null\": \"j\"}'),('1','{\"col_int_undef_signed\": 9, \"col_in [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"do\", \"col_varchar_1024__undef_signed\": \"but\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"c\", \"col_varchar_1024__undef_signed\": \"would\"}'),('2','{\"col_int_undef_signed\": 3, \"col_varchar_10__undef_signed\": \"l\", \"col_varchar_1024__undef_signed\": \"x\"}'),('3','{\"col_int_undef_si [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"out\", \"col_varchar_1024__undef_signed\": \"ok\"}'),('1','{\"col_int_undef_signed\": 5, \"col_varchar_10__undef_signed\": \"see\", \"col_varchar_1024__undef_signed\": \"a\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"b\", \"col_varchar_1024__undef_signed\": \"s\"}'),('3','{\"col_int_undef_sign [...] +INSERT INTO table_10_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"q\", \"col_varchar_1024__undef_signed\": \"like\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"j\", \"col_varchar_1024__undef_signed\": \"n\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"l\", \"col_varchar_1024__undef_signed\": \"a\"}'),('3','{\"col_int_undef_sig [...] +INSERT INTO table_21_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 0, \"col_varchar_10__undef_signed\": \"it\", \"col_varchar_1024__undef_signed\": \"oh\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"then\", \"col_varchar_1024__undef_signed\": \"x\"}'),('2','{\"col_int_undef_signed\": 0, \"col_varchar_10__undef_signed\": \"didn\'t\", \"col_varchar_1024__undef_signed\": \"her\"}'),('3','{\"col_int_undef [...] +INSERT INTO table_21_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 0, \"col_varchar_10__undef_signed\": \"why\", \"col_varchar_1024__undef_signed\": \"t\"}'),('1','{\"col_int_undef_signed\": 2, \"col_varchar_10__undef_signed\": \"h\", \"col_varchar_1024__undef_signed\": \"from\"}'),('2','{\"col_int_undef_signed\": 3, \"col_varchar_10__undef_signed\": \"s\", \"col_varchar_1024__undef_signed\": \"v\"}'),('3','{\"col_int_undef_signed\": [...] +INSERT INTO table_21_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 9, \"col_varchar_10__undef_signed\": \"j\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"p\"}'),('2','{\"col_int_undef_signed\": 4, \"col_varchar_10__undef_signed\": \"was\"}'),('3','{\"col_int_undef_signed\": 9, \"col_varchar_10__undef_signed\": \"a\"}'),('4','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"\"}'),(' [...] +INSERT INTO table_21_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 5, \"col_date_undef_signed\": \"2023-12-19\", \"col_date_undef_signed_not_null\": \"2023-12-16\", \"col_varchar_10__undef_signed\": \"okay\", \"col_varchar_10__undef_signed_not_null\": \"know\", \"col_varchar_1024__undef_signed\": \"d\", \"col_varchar_1024__undef_signed_not_null\": \"y\"}'),('1','{\"col_int_undef_signed\": 5, \" [...] +INSERT INTO table_21_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 8, \"col_varchar_10__undef_signed\": null, \"col_varchar_10__undef_signed_not_null\": \"mean\"}'),('1','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 8, \"col_varchar_10__undef_signed\": \"is\", \"col_varchar_10__undef_signed_not_null\": \"l\"}'),('2','{\"col_int_undef_signed\": null, \"col_int_undef_signed [...] +INSERT INTO table_21_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"good\", \"col_varchar_1024__undef_signed\": \"y\"}'),('1','{\"col_int_undef_signed\": 3, \"col_varchar_10__undef_signed\": \"f\", \"col_varchar_1024__undef_signed\": \"it\"}'),('2','{\"col_int_undef_signed\": 4, \"col_varchar_10__undef_signed\": \"d\", \"col_varchar_1024__undef_signed\": \"l\"}'),('3','{\"col_int_undef_signed\" [...] +INSERT INTO table_25_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 2, \"col_int_undef_signed_not_null\": 4, \"col_varchar_10__undef_signed\": \"I\'m\", \"col_varchar_10__undef_signed_not_null\": \"\"}'),('1','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 8, \"col_varchar_10__undef_signed\": \"\", \"col_varchar_10__undef_signed_not_null\": \"w\"}'),('2','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_ [...] +INSERT INTO table_25_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 7, \"col_int_undef_signed_not_null\": 5, \"col_date_undef_signed\": \"2023-12-18\", \"col_date_undef_signed_not_null\": \"2023-12-11\", \"col_varchar_10__undef_signed\": \"v\", \"col_varchar_10__undef_signed_not_null\": \"r\", \"col_varchar_1024__undef_signed\": \"want\", \"col_varchar_1024__undef_signed_not_null\": \"do\"}'),('1','{\"col_int_undef_signed\": null, \"co [...] +INSERT INTO table_25_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 7, \"col_varchar_10__undef_signed\": \"k\", \"col_varchar_1024__undef_signed\": \"v\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"okay\", \"col_varchar_1024__undef_signed\": \"e\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"and\", \"col_varchar_1024__undef_signed\": \"g\"}'),('3','{\"col_int_undef_sign [...] +INSERT INTO table_25_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"c\", \"col_varchar_1024__undef_signed\": \"s\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"yes\", \"col_varchar_1024__undef_signed\": \"can\"}'),('2','{\"col_int_undef_signed\": 2, \"col_varchar_10__undef_signed\": \"y\", \"col_varchar_1024__undef_signed\": \"m\"}'),('3','{\"col_int_undef_signe [...] +INSERT INTO table_25_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"for\", \"col_varchar_1024__undef_signed\": \"can\'t\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"know\", \"col_varchar_1024__undef_signed\": \"well\"}'),('2','{\"col_int_undef_signed\": 0, \"col_varchar_10__undef_signed\": \"up\", \"col_varchar_1024__undef_signed\": \"v\"}'),('3','{\"col_int_u [...] +INSERT INTO table_25_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"y\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"tell\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"because\"}'),('3','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"\"}'),('4','{\"col_int_undef_signed\": 2, \"col_varchar_10__undef_signed\" [...] +INSERT INTO table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 0, \"col_varchar_10__undef_signed\": \"something\", \"col_varchar_1024__undef_signed\": \"f\"}'),('1','{\"col_int_undef_signed\": 0, \"col_varchar_10__undef_signed\": \"really\", \"col_varchar_1024__undef_signed\": \"g\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"h\", \"col_varchar_1024__undef_signed\": \"of\"}'),('3','{\"col_int_undef [...] +INSERT INTO table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 5, \"col_int_undef_signed_not_null\": 7, \"col_varchar_10__undef_signed\": \"k\", \"col_varchar_10__undef_signed_not_null\": \"\"}'),('1','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 7, \"col_varchar_10__undef_signed\": \"how\", \"col_varchar_10__undef_signed_not_null\": \"\"}'),('2','{\"col_int_undef_signed\": 9, \"col_int_undef_signed_not_null\ [...] +INSERT INTO table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 0, \"col_date_undef_signed\": \"2023-12-11\", \"col_date_undef_signed_not_null\": \"2023-12-13\", \"col_varchar_10__undef_signed\": \"who\", \"col_varchar_10__undef_signed_not_null\": \"back\", \"col_varchar_1024__undef_signed\": \"t\", \"col_varchar_1024__undef_signed_not_null\": \"j\"}'),('1','{\"col_int_undef_signed\": 9, \"co [...] +INSERT INTO table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"could\", \"col_varchar_1024__undef_signed\": \"want\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"r\", \"col_varchar_1024__undef_signed\": \"mean\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"will\", \"col_varchar_1024__undef_signed\": \"your\"}'),('3','{\"col_i [...] +INSERT INTO table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"think\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"\"}'),('2','{\"col_int_undef_signed\": 2, \"col_varchar_10__undef_signed\": \"\"}'),('3','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"r\"}'),('4','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": null} [...] +INSERT INTO table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"this\", \"col_varchar_1024__undef_signed\": \"think\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"l\", \"col_varchar_1024__undef_signed\": \"as\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"how\", \"col_varchar_1024__undef_signed\": \"o\"}'),('3','{\"col_int_und [...] +INSERT INTO table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 1, \"col_int_undef_signed_not_null\": 1, \"col_date_undef_signed\": \"2023-12-12\", \"col_date_undef_signed_not_null\": \"2023-12-18\", \"col_varchar_10__undef_signed\": \"c\", \"col_varchar_10__undef_signed_not_null\": \"d\", \"col_varchar_1024__undef_signed\": \"d\", \"col_varchar_1024__undef_signed_not_null\": \"f\"}'),('1','{\"col_int_undef_signed\": null, \"col_int [...] +INSERT INTO table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 7, \"col_varchar_10__undef_signed\": \"y\", \"col_varchar_1024__undef_signed\": \"g\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"t\", \"col_varchar_1024__undef_signed\": \"can\"}'),('2','{\"col_int_undef_signed\": 8, \"col_varchar_10__undef_signed\": \"would\", \"col_varchar_1024__undef_signed\": \"l\"}'),('3','{\"col_int_undef_signed\ [...] +INSERT INTO table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": 8, \"col_varchar_10__undef_signed\": \"w\", \"col_varchar_1024__undef_signed\": \"a\"}'),('1','{\"col_int_undef_signed\": 8, \"col_varchar_10__undef_signed\": \"k\", \"col_varchar_1024__undef_signed\": \"c\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"her\", \"col_varchar_1024__undef_signed\": \"come\"}'),('3','{\"col_int_undef_signed\" [...] +INSERT INTO table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 0, \"col_varchar_10__undef_signed\": \"didn\'t\", \"col_varchar_10__undef_signed_not_null\": \"d\"}'),('1','{\"col_int_undef_signed\": null, \"col_int_undef_signed_not_null\": 1, \"col_varchar_10__undef_signed\": null, \"col_varchar_10__undef_signed_not_null\": \"\"}'),('2','{\"col_int_undef_signed\": 9, \"col_int_undef_signed_no [...] +INSERT INTO table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"q\"}'),('1','{\"col_int_undef_signed\": 7, \"col_varchar_10__undef_signed\": \"k\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": null}'),('3','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"\"}'),('4','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"\"}'),( [...] +INSERT INTO table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"this\", \"col_varchar_1024__undef_signed\": \"who\"}'),('1','{\"col_int_undef_signed\": 1, \"col_varchar_10__undef_signed\": \"x\", \"col_varchar_1024__undef_signed\": \"you\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"didn\'t\", \"col_varchar_1024__undef_signed\": \"this\"}'),('3','{\"col_int_ [...] +SELECT t1.pk AS pk1, CAST(t1 . var['col_int_undef_signed'] AS int) AS col1, CAST(t1 . var['col_varchar_10__undef_signed'] AS varchar(10)) AS col2, t2.pk AS pk1, CAST(t2 . var['col_int_undef_signed'] AS int) AS col3, CAST(t2 . var['col_varchar_10__undef_signed'] AS varchar(10)) AS col4 FROM table_25_undef_partitions2_keys3_properties4_distributed_by5 AS t1 LEFT OUTER JOIN table_21_undef_partitions2_keys3_properties4_distributed_by5 AS t2 ON t2 . pk < 0 AND t2 . pk = 8 INNER JOIN [...] diff --git a/regression-test/suites/variant_p0/rqg/rqg4.sql b/regression-test/suites/variant_p0/rqg/rqg4.sql new file mode 100644 index 00000000000..775bb2a576a --- /dev/null +++ b/regression-test/suites/variant_p0/rqg/rqg4.sql @@ -0,0 +1,6 @@ +CREATE TABLE table_500_undef_partitions2_keys3_properties4_distributed_by52 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_500_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_int_undef_signed\": 1, \"col_int_undef_signed2\": 2, \"col_date_undef_signed\": \"2025-06-18\", \"col_date_undef_signed2\": \"2024-02-18\", \"col_varchar_10__undef_signed\": \"i\", \"col_varchar_1024__undef_signed\": \"i\"}'),('1','{\"col_int_undef_signed\": 0, \"col_int_undef_signed2\": 5, \"col_date_undef_signed\": \"2026-02-18\", \"col_date_undef_signed2\": \"2023-12-19\", \"col_varc [...] +INSERT INTO table_500_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_int_undef_signed\": 6, \"col_int_undef_signed2\": 7, \"col_date_undef_signed\": \"2023-12-17\", \"col_date_undef_signed2\": \"2023-12-20\", \"col_varchar_10__undef_signed\": \"j\", \"col_varchar_1024__undef_signed\": \"u\"}'),('1','{\"col_int_undef_signed\": 1, \"col_int_undef_signed2\": 3, \"col_date_undef_signed\": \"2025-02-18\", \"col_date_undef_signed2\": \"2023-12-16\", \"col_varc [...] + +SELECT COUNT() AS field1 FROM table_500_undef_partitions2_keys3_properties4_distributed_by52 AS table1 WHERE ( CAST(table1 . var['col_date_undef_signed'] AS text) IN ( '2027-01-16', '2023-12-15', '2023-12-15' )); +SELECT COUNT() AS field1 FROM table_500_undef_partitions2_keys3_properties4_distributed_by52 AS table1 WHERE ( CAST(table1 . var['col_date_undef_signed'] AS datetime) IN ( '2027-01-16', '2023-12-15', '2023-12-15' )); \ No newline at end of file diff --git a/regression-test/suites/variant_p0/rqg/rqg5.sql b/regression-test/suites/variant_p0/rqg/rqg5.sql new file mode 100644 index 00000000000..c19d799d40d --- /dev/null +++ b/regression-test/suites/variant_p0/rqg/rqg5.sql @@ -0,0 +1,22 @@ +CREATE TABLE table_50_undef_partitions2_keys3_properties4_distributed_by53 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_bigint_undef_signed\": null, \"col_bigint_undef_signed2\": 18332}'),('1','{\"col_bigint_undef_signed\": 788547, \"col_bigint_undef_signed2\": null}'),('2','{\"col_bigint_undef_signed\": 4644959, \"col_bigint_undef_signed2\": -56}'),('3','{\"col_bigint_undef_signed\": 8364628, \"col_bigint_undef_signed2\": 72}'),('4','{\"col_bigint_undef_signed\": null, \"col_bigint_undef_signed2\": -5581 [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": 2, \"col_date_undef_signed\": \"2011-03-24\", \"col_datetime_undef_signed\": \"2009-09-17 17:25:39\"}'),('1','{\"col_int_undef_signed\": null, \"col_date_undef_signed\": \"2023-12-10\", \"col_datetime_undef_signed\": \"2001-09-19 20:24:22\"}'),('2','{\"col_int_undef_signed\": 1, \"col_date_undef_signed\": null, \"col_datetime_undef_signed\": \"2003-04-04 02:59:52\"}') [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": 2, \"col_int_undef_signed_not_null\": -10, \"col_bigint_undef_signed\": -5139255822006861284, \"col_bigint_undef_signed_not_null\": 8767609723338481716, \"col_date_undef_signed\": \"2027-01-09\", \"col_date_undef_signed_not_null\": \"2023-12-11\", \"col_varchar_10__undef_signed\": \"u\", \"col_varchar_10__undef_signed_not_null\": \"o\", \"col_varchar_1024__undef_signe [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": 2, \"col_int_undef_signed_not_null\": 4, \"col_date_undef_signed\": \"2023-12-17\", \"col_date_undef_signed_not_null\": \"2023-12-14\", \"col_varchar_5__undef_signed\": \"q\", \"col_varchar_5__undef_signed_not_null\": \"t\"}'),('1','{\"col_int_undef_signed\": 3, \"col_int_undef_signed_not_null\": 4, \"col_date_undef_signed\": \"2023-12-16\", \"col_date_undef_signed_no [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_int_undef_signed_index_inverted\": null, \"col_int_undef_signed_not_null\": -10, \"col_int_undef_signed_not_null_index_inverted\": -4, \"col_bigint_undef_signed\": null, \"col_bigint_undef_signed_index_inverted\": -8106433477491953829, \"col_bigint_undef_signed_not_null\": -4802033018054435794, \"col_bigint_undef_signed_not_null_index_inverted\": -59924112 [...] +CREATE TABLE table_50_undef_partitions2_keys3_properties4_distributed_by520 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +CREATE TABLE table_50_undef_partitions2_keys3_properties4_distributed_by52 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by520(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_int_undef_signed_index_inverted\": 6, \"col_int_undef_signed_not_null\": 2, \"col_int_undef_signed_not_null_index_inverted\": -10, \"col_bigint_undef_signed\": 9011200852160137244, \"col_bigint_undef_signed_index_inverted\": -4304767207691790360, \"col_bigint_undef_signed_not_null\": 6062089241742974193, \"col_bigint_undef_signed_not_null_index_inverted\" [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_bigint_undef_signed\": 15, \"col_bigint_undef_signed2\": 103}'),('1','{\"col_bigint_undef_signed\": null, \"col_bigint_undef_signed2\": 3488}'),('2','{\"col_bigint_undef_signed\": -4865121, \"col_bigint_undef_signed2\": -2111291}'),('3','{\"col_bigint_undef_signed\": 2217427, \"col_bigint_undef_signed2\": 53}'),('4','{\"col_bigint_undef_signed\": -195, \"col_bigint_undef_signed2\": 97}') [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_int_undef_signed\": -10, \"col_int_undef_signed_index_inverted\": -10, \"col_int_undef_signed_not_null\": -4, \"col_int_undef_signed_not_null_index_inverted\": -4, \"col_bigint_undef_signed\": 6921102597338769372, \"col_bigint_undef_signed_index_inverted\": -7846920156574865363, \"col_bigint_undef_signed_not_null\": -4080470861525976092, \"col_bigint_undef_signed_not_null_index_inverted\ [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_int_undef_signed\": -10, \"col_int_undef_signed_not_null\": 1, \"col_bigint_undef_signed\": -9117079895128902283, \"col_bigint_undef_signed_not_null\": -3196878189683372171, \"col_date_undef_signed\": null, \"col_date_undef_signed_not_null\": \"2023-12-15\", \"col_varchar_10__undef_signed\": \"m\", \"col_varchar_10__undef_signed_not_null\": \"c\", \"col_varchar_1024__undef_signed\": \"s\ [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_int_undef_signed\": 1, \"col_date_undef_signed\": \"2023-12-15\", \"col_datetime_undef_signed\": \"2005-01-21 14:56:21\"}'),('1','{\"col_int_undef_signed\": 9, \"col_date_undef_signed\": \"2023-12-16\", \"col_datetime_undef_signed\": \"2017-11-01 12:05:26\"}'),('2','{\"col_int_undef_signed\": 1, \"col_date_undef_signed\": \"2023-12-18\", \"col_datetime_undef_signed\": \"2005-12-04 00:49: [...] +INSERT INTO table_50_undef_partitions2_keys3_properties4_distributed_by52(pk,var) VALUES ('0','{\"col_int_undef_signed\": 3, \"col_int_undef_signed_not_null\": 9, \"col_date_undef_signed\": \"2023-12-11\", \"col_date_undef_signed_not_null\": \"2023-12-11\", \"col_varchar_5__undef_signed\": \"h\", \"col_varchar_5__undef_signed_not_null\": \"l\"}'),('1','{\"col_int_undef_signed\": 2, \"col_int_undef_signed_not_null\": 5, \"col_date_undef_signed\": \"2023-12-10\", \"col_date_undef_signed_no [...] +CREATE TABLE table_100_undef_partitions2_keys3_properties4_distributed_by53 ( pk int, var VARIANT NULL ) engine=olap DUPLICATE KEY(pk) distributed by hash(pk) buckets 10 properties("replication_num" = "1"); +INSERT INTO table_100_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_bigint_undef_signed\": 28862, \"col_bigint_undef_signed2\": -25225}'),('1','{\"col_bigint_undef_signed\": 5516858, \"col_bigint_undef_signed2\": -4609390}'),('2','{\"col_bigint_undef_signed\": -7815300, \"col_bigint_undef_signed2\": null}'),('3','{\"col_bigint_undef_signed\": null, \"col_bigint_undef_signed2\": -7685824}'),('4','{\"col_bigint_undef_signed\": 22293, \"col_bigint_undef_si [...] +INSERT INTO table_100_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_bigint_undef_signed\": 55, \"col_bigint_undef_signed2\": -58}'),('1','{\"col_bigint_undef_signed\": 49, \"col_bigint_undef_signed2\": 29792}'),('2','{\"col_bigint_undef_signed\": 95, \"col_bigint_undef_signed2\": 3551878}'),('3','{\"col_bigint_undef_signed\": 7833628, \"col_bigint_undef_signed2\": -6983097}'),('4','{\"col_bigint_undef_signed\": -27400, \"col_bigint_undef_signed2\": null [...] +INSERT INTO table_100_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": 1, \"col_varchar_10__undef_signed\": \"k\", \"col_varchar_1024__undef_signed\": \"t\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"b\", \"col_varchar_1024__undef_signed\": \"r\"}'),('2','{\"col_int_undef_signed\": 6, \"col_varchar_10__undef_signed\": \"then\", \"col_varchar_1024__undef_signed\": \"n\"}'),('3','{\"col_int_undef_signed\ [...] +INSERT INTO table_100_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": 1, \"col_varchar_10__undef_signed\": \"p\", \"col_varchar_1024__undef_signed\": \"why\"}'),('1','{\"col_int_undef_signed\": 4, \"col_varchar_10__undef_signed\": \"didn\'t\", \"col_varchar_1024__undef_signed\": \"got\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"will\", \"col_varchar_1024__undef_signed\": \"z\"}'),('3','{\"col_int_und [...] +INSERT INTO table_100_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": 6, \"col_int_undef_signed_not_null\": 6, \"col_date_undef_signed\": \"2023-12-11\", \"col_date_undef_signed_not_null\": \"2023-12-09\", \"col_varchar_10__undef_signed\": \"u\", \"col_varchar_10__undef_signed_not_null\": \"g\", \"col_varchar_1024__undef_signed\": \"y\", \"col_varchar_1024__undef_signed_not_null\": \"well\"}'),('1','{\"col_int_undef_signed\": 4, \"col_ [...] +INSERT INTO table_100_undef_partitions2_keys3_properties4_distributed_by53(pk,var) VALUES ('0','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"u\", \"col_varchar_1024__undef_signed\": \"the\"}'),('1','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"c\", \"col_varchar_1024__undef_signed\": \"w\"}'),('2','{\"col_int_undef_signed\": null, \"col_varchar_10__undef_signed\": \"he\'s\", \"col_varchar_1024__undef_signed\": \"really\"}'),('3','{\"col_int [...] +SELECT T1.pk AS C1 FROM table_50_undef_partitions2_keys3_properties4_distributed_by53 AS T1 LEFT OUTER JOIN table_50_undef_partitions2_keys3_properties4_distributed_by52 AS T2 ON CAST(T1 . var['col_bigint_undef_signed2'] AS bigint) = CAST(T2 . var['col_bigint_undef_signed2'] AS bigint) AND CAST(T1 . var['col_bigint_undef_signed'] AS bigint) < (SELECT AVG ( CAST(T3 . var['col_bigint_undef_signed2'] AS bigint) ) FROM table_100_undef_partitions2_keys3_properties4_distribu [...] \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org