eldenmoon commented on code in PR #65561: URL: https://github.com/apache/doris/pull/65561#discussion_r3610669061
########## be/src/core/column/variant_v2/column_variant_v2.cpp: ########## @@ -0,0 +1,1604 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/variant_v2/column_variant_v2.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <string_view> +#include <typeinfo> +#include <utility> + +#include "common/check.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_decimal.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/column/columns_common.h" +#include "core/column/variant_v2/column_variant_v2_typed_scalar.h" +#include "core/custom_allocator.h" +#include "util/hash_util.hpp" +#include "util/variant/variant_canonical.h" +#include "util/variant/variant_encoded_block.h" +#include "util/variant/variant_scalar_encoding.h" + +namespace doris { +namespace { + +using MetaIdsColumn = ColumnVector<TYPE_UINT32>; +using column_variant_v2_internal::TypedFallbackKind; +using column_variant_v2_internal::dispatch_typed_column; +using column_variant_v2_internal::exact_typed_identity; +using column_variant_v2_internal::is_supported_typed_identity; +using column_variant_v2_internal::validate_typed_decimal_scale; +using column_variant_v2_internal::visit_typed_canonical_rows; +using column_variant_v2_internal::visit_typed_rows; +constexpr uint32_t UNMAPPED_METADATA_ID = std::numeric_limits<uint32_t>::max(); +constexpr size_t CANONICAL_SIZE_PREFIX = sizeof(uint32_t); +constexpr std::array<char, 3> EMPTY_OBJECT_METADATA {static_cast<char>(0x11), 0, 0}; +constexpr std::array<char, 3> EMPTY_OBJECT_VALUE {static_cast<char>(0x02), 0, 0}; + +uint32_t read_canonical_payload_size(const char* pos) { + DCHECK(pos != nullptr); + uint32_t payload_size = 0; + for (uint8_t byte = 0; byte < CANONICAL_SIZE_PREFIX; ++byte) { + payload_size |= static_cast<uint32_t>(static_cast<uint8_t>(pos[byte])) << (byte * 8); + } + return payload_size; +} + +size_t trusted_canonical_cell_size(const char* pos) { + return CANONICAL_SIZE_PREFIX + read_canonical_payload_size(pos); +} + +void check_hash_range(const ColumnVariantV2& column, size_t start, size_t end) { + DCHECK_LE(start, end); + DCHECK_LE(end, column.size()); +} + +template <typename Sink, typename Hash> +void update_canonical_hash_range(const ColumnVariantV2& column, size_t start, size_t end, + Hash& hash, const uint8_t* __restrict null_data) { + check_hash_range(column, start, end); + Sink sink(hash); + for (size_t row = start; row < end; ++row) { + if (null_data == nullptr || null_data[row] == 0) { + canonical_hash(column.get_value_ref(row), sink); + } + } + hash = sink.digest(); +} + +template <typename Sink, typename Hash> +void update_canonical_hashes(const ColumnVariantV2& column, Hash* __restrict hashes, + const uint8_t* __restrict null_data) { + for (size_t row = 0; row < column.size(); ++row) { + if (null_data == nullptr || null_data[row] == 0) { + Sink sink(hashes[row]); + canonical_hash(column.get_value_ref(row), sink); + hashes[row] = sink.digest(); + } + } +} + +void validate_offsets(StringRef bytes, std::span<const uint32_t> offsets, + std::string_view description) { + DORIS_CHECK(bytes.data != nullptr || bytes.size == 0) + << description << " bytes have a null pointer"; + DORIS_CHECK(!offsets.empty()) << description << " offsets cannot be empty"; + DORIS_CHECK_EQ(offsets.front(), 0) << description << " offsets must start at zero"; + DORIS_CHECK_EQ(static_cast<size_t>(offsets.back()), bytes.size) + << description << " offsets must end at the byte size"; + for (size_t index = 1; index < offsets.size(); ++index) { + DORIS_CHECK_LT(offsets[index - 1], offsets[index]) + << description << " offsets must be strictly increasing"; + } +} + +void require_exclusive(const IColumn::WrappedPtr& column, std::string_view description) { + const auto& immutable = static_cast<const IColumn::Ptr&>(column); + DORIS_CHECK(immutable->is_exclusive()) + << "ColumnVariantV2 " << description << " must be COW-detached before mutation"; +} + +void reserve_rows(ColumnString& values, MetaIdsColumn& metadata_ids, size_t value_bytes, + size_t rows) { + const size_t final_value_bytes = values.get_chars().size() + value_bytes; + ColumnString::check_chars_length(final_value_bytes, values.size() + rows, values.size()); + values.get_chars().reserve(final_value_bytes); + values.get_offsets().reserve(values.size() + rows); + metadata_ids.get_data().reserve(metadata_ids.size() + rows); +} + +void append_metadata_ids(MetaIdsColumn& destination, const DorisVector<uint32_t>& ids) { + auto& data = destination.get_data(); + const size_t old_size = data.size(); + data.resize(old_size + ids.size()); + std::ranges::copy(ids, data.begin() + old_size); +} + +struct SelectedRows { + DorisVector<uint32_t> metadata_ids; + size_t value_bytes = 0; +}; + +size_t validate_selected_indices(const uint32_t* indices_begin, const uint32_t* indices_end, + size_t source_rows) { + if (indices_begin == indices_end) { + return 0; + } + DORIS_CHECK(indices_begin != nullptr && indices_end != nullptr) + << "non-empty source indices cannot be null"; + DORIS_CHECK_LT(indices_begin, indices_end) << "source index range is reversed"; + const size_t rows = indices_end - indices_begin; + DORIS_CHECK_LT(*std::max_element(indices_begin, indices_end), source_rows) + << "source index is out of range"; + return rows; +} + +DorisVector<uint32_t> collect_range_metadata_ids( + const MetaIdsColumn::Container& source_metadata_ids, size_t source_metadata_count, + size_t start, size_t length) { + DorisVector<uint32_t> selected(length); + for (size_t row = 0; row < length; ++row) { + const uint32_t metadata_id = source_metadata_ids[start + row]; + DCHECK_LT(metadata_id, source_metadata_count) << "source metadata id is out of range"; + selected[row] = metadata_id; + } + return selected; +} + +SelectedRows collect_selected_rows(const MetaIdsColumn::Container& source_metadata_ids, + const ColumnString& source_metadatas, + const ColumnString& source_values, const uint32_t* indices_begin, + size_t rows) { + SelectedRows selected; + selected.metadata_ids.resize(rows); + for (size_t row = 0; row < rows; ++row) { + const uint32_t source_row = indices_begin[row]; + const uint32_t metadata_id = source_metadata_ids[source_row]; + DCHECK_LT(metadata_id, source_metadatas.size()) << "source metadata id is out of range"; + selected.metadata_ids[row] = metadata_id; + const size_t row_bytes = source_values.get_data_at(source_row).size; + DORIS_CHECK_LE(row_bytes, std::numeric_limits<size_t>::max() - selected.value_bytes) + << "selected Variant value bytes overflow size_t"; + selected.value_bytes += row_bytes; + } + return selected; +} + +template <typename Sink, typename Hash> +void update_typed_hashes(const ColumnNullable& nullable, PrimitiveType type, uint32_t scale, + Hash* __restrict hashes, const uint8_t* __restrict null_data) { + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + visit_typed_canonical_rows<Type>(nullable, column, scale, 0, nullable.size(), + [&](size_t row, auto&& canonical_factory) { + if (null_data == nullptr || null_data[row] == 0) { + VariantCanonicalScalarRef canonical = + canonical_factory(); + Sink sink(hashes[row]); + canonical_hash(canonical, sink); + hashes[row] = sink.digest(); + } + }); + }); +} + +template <typename Sink, typename Hash> +void update_typed_hash_range(const ColumnNullable& nullable, PrimitiveType type, uint32_t scale, + size_t start, size_t end, Hash& hash, + const uint8_t* __restrict null_data) { + Sink sink(hash); + dispatch_typed_column(nullable, type, [&]<PrimitiveType Type>(const auto& column) { + visit_typed_canonical_rows<Type>( + nullable, column, scale, start, end, [&](size_t row, auto&& canonical_factory) { + if (null_data == nullptr || null_data[row] == 0) { + VariantCanonicalScalarRef canonical = canonical_factory(); + canonical_hash(canonical, sink); + } + }); + }); + hash = sink.digest(); +} + +struct TypedEncodingResult { + ColumnString::MutablePtr metadatas; + MetaIdsColumn::MutablePtr metadata_ids; + ColumnString::MutablePtr values; + ColumnVariantV2::TypedEncodingStats stats; +}; + +void count_fallback(TypedFallbackKind fallback, ColumnVariantV2::TypedEncodingStats& stats) { + switch (fallback) { + case TypedFallbackKind::NONE: + return; + case TypedFallbackKind::LARGEINT: + ++stats.largeint_string_fallback_rows; + return; + case TypedFallbackKind::IP: + ++stats.ip_string_fallback_rows; + return; + } +} + +template <PrimitiveType Type, typename Column> +TypedEncodingResult encode_typed_column(const ColumnNullable& nullable, const Column& column, + uint32_t scale) { + size_t value_bytes = 0; + TypedEncodingResult result; + visit_typed_rows<Type>( Review Comment: 这个visit_typed_rows是否只是收集统计信息和确认是否char溢出， 能不能只能单测用， 可能影响性能 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
