eldenmoon commented on code in PR #65561: URL: https://github.com/apache/doris/pull/65561#discussion_r3619268247
########## be/src/core/data_type_serde/data_type_variant_v2_serde_binary.cpp: ########## @@ -0,0 +1,486 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <algorithm> +#include <limits> +#include <new> +#include <string_view> + +#include "common/exception.h" +#include "core/column/column_const.h" +#include "core/data_type_serde/data_type_variant_v2_serde.h" +#include "core/data_type_serde/data_type_variant_v2_serde_binary_internal.h" +#include "core/value/variant/variant_field.h" + +namespace doris::variant_v2_serde_binary_internal { + +size_t checked_add(size_t left, size_t right, std::string_view description) { + if (right > std::numeric_limits<size_t>::max() - left) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Variant V2 exchange {} exceeds size_t", + description); + } + return left + right; +} + +size_t checked_multiply(size_t left, size_t right, std::string_view description) { + if (left != 0 && right > std::numeric_limits<size_t>::max() / left) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Variant V2 exchange {} exceeds size_t", + description); + } + return left * right; +} + +uint32_t checked_u32(size_t value, std::string_view description) { + if (value > std::numeric_limits<uint32_t>::max()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Variant V2 exchange {} size {} exceeds uint32", description, value); + } + return static_cast<uint32_t>(value); +} + +size_t checked_size(uint64_t value, std::string_view description) { + if (value > std::numeric_limits<size_t>::max()) { + throw Exception(ErrorCode::CORRUPTION, + "Variant V2 exchange {} size {} exceeds addressable memory", description, + value); + } + return static_cast<size_t>(value); +} + +namespace { + +constexpr uint32_t UNMAPPED_METADATA = std::numeric_limits<uint32_t>::max(); + +struct EncodedPlan { + uint32_t metadata_offsets_bytes = 0; + uint32_t metadata_bytes = 0; + uint32_t meta_ids_bytes = 0; + uint32_t value_offsets_bytes = 0; + uint32_t values_bytes = 0; + size_t frame_bytes = 0; + DorisVector<uint32_t> metadata_offsets; + DorisVector<VariantMetadataRef> metadatas; + DorisVector<uint32_t> meta_ids; + DorisVector<uint32_t> value_offsets; + DorisVector<VariantRef> values; +}; + +ResolvedColumn resolve_column(const IColumn& source) { + const auto* constant = check_and_get_column<ColumnConst>(&source); + const IColumn& physical = constant == nullptr ? source : constant->get_data_column(); + if (typeid(physical) != typeid(ColumnVariantV2)) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Variant V2 exchange requires an exact ColumnVariantV2 or ColumnConst " + "thereof"); + } + const auto& variant = assert_cast<const ColumnVariantV2&>(physical); + if (constant != nullptr && variant.size() != 1) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "constant Variant V2 source must contain exactly one physical row, found " + "{}", + variant.size()); + } + if (constant == nullptr && variant.size() != source.size()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Variant V2 source logical size {} does not match physical size {}", + source.size(), variant.size()); + } + return {.view = variant.read_view(), + .logical_rows = source.size(), + .is_constant = constant != nullptr}; +} + +EncodedPlan plan_encoded(const ResolvedColumn& source) { + EncodedPlan plan; + const size_t source_metadata_count = source.view.metadata_count(); + DorisVector<uint32_t> source_to_dense(source_metadata_count, UNMAPPED_METADATA); + plan.metadata_offsets.push_back(0); + plan.meta_ids.reserve(source.logical_rows); + plan.value_offsets.push_back(0); + plan.values.reserve(source.logical_rows); + + size_t metadata_bytes = 0; + size_t values_bytes = 0; + for (size_t row = 0; row < source.logical_rows; ++row) { + const size_t physical_row = source.physical_row(row); + const uint32_t source_id = source.view.metadata_id_at(physical_row); + DORIS_CHECK_LT(source_id, source_to_dense.size()); + uint32_t dense_id = source_to_dense[source_id]; + if (dense_id == UNMAPPED_METADATA) { + const VariantMetadataRef metadata = source.view.metadata_at(source_id); + validate_variant_metadata(metadata); + dense_id = checked_u32(plan.metadatas.size(), "metadata count"); + source_to_dense[source_id] = dense_id; + plan.metadatas.push_back(metadata); + metadata_bytes = checked_add(metadata_bytes, metadata.size, "metadata bytes"); + plan.metadata_offsets.push_back(checked_u32(metadata_bytes, "metadata bytes")); Review Comment: 为什么需要序列化metadata_offsets， 直接分别序列化_metadatas / _meta_ids不就够了吗？ -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
