github-actions[bot] commented on code in PR #15309: URL: https://github.com/apache/doris/pull/15309#discussion_r1055999472
########## be/src/vec/utils/histogram_helpers.hpp: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <rapidjson/document.h> +#include <rapidjson/prettywriter.h> +#include <rapidjson/stringbuffer.h> + +#include <boost/dynamic_bitset.hpp> + +#include "vec/data_types/data_type_decimal.h" + +namespace doris::vectorized { + +template <typename T> +struct Bucket { +public: + Bucket() = default; + Bucket(T value, size_t pre_sum) + : lower(value), upper(value), count(1), pre_sum(pre_sum), ndv(1) {} + Bucket(T lower, T upper, size_t count, size_t pre_sum, size_t ndv) + : lower(lower), upper(upper), count(count), pre_sum(pre_sum), ndv(ndv) {} + + T lower; + T upper; + uint64_t count; + uint64_t pre_sum; + uint64_t ndv; +}; + +template <typename T> +bool build_bucket_from_data(std::vector<Bucket<T>>& buckets, + const std::map<T, uint64_t>& ordered_map, double sample_rate, + uint32_t max_bucket_num) { + if (ordered_map.size() == 0) { + return false; + } + + uint64_t element_number = 0; + for (auto it : ordered_map) { + element_number += it.second; + } + + auto sample_number = (uint64_t)std::ceil(element_number * sample_rate); + auto num_per_bucket = (uint64_t)std::ceil((Float64)sample_number / max_bucket_num); + + LOG(INFO) << fmt::format( + "histogram bucket info: element number {}, sample number:{}, num per bucket:{}", + element_number, sample_number, num_per_bucket); + + if (sample_rate == 1) { + for (auto it : ordered_map) { + for (auto i = it.second; i > 0; --i) { + auto v = it.first; + value_to_bucket(buckets, v, num_per_bucket); + } + } + return true; + } + + // if sampling is required (0<sample_rate<1), + // we need to build the sampled data index + boost::dynamic_bitset<> sample_index(element_number); + + // use a same seed value so that we get + // same result each time we run this function + srand(element_number * sample_rate * max_bucket_num); + + while (sample_index.count() < sample_number) { + uint64_t num = (rand() % (element_number)); + sample_index[num] = true; + } + + uint64_t element_cnt = 0; + uint64_t sample_cnt = 0; + bool break_flag = false; + + for (auto it : ordered_map) { + if (break_flag) { + break; + } + for (auto i = it.second; i > 0; --i) { + if (sample_cnt >= sample_number) { + break_flag = true; + break; + } + if (sample_index[element_cnt]) { + sample_cnt += 1; + auto v = it.first; + value_to_bucket(buckets, v, num_per_bucket); + } + element_cnt += 1; + } + } + + return true; +} + +bool inline bucket_to_json(rapidjson::Document::AllocatorType& allocator, + rapidjson::Value& bucket_arr, std::string lower, std::string upper, + int64 count, int64 pre_sum, int64 ndv) { + rapidjson::Value bucket(rapidjson::kObjectType); + + rapidjson::Value lower_val(lower.c_str(), allocator); + bucket.AddMember("lower", lower_val, allocator); + + rapidjson::Value upper_val(upper.c_str(), allocator); + bucket.AddMember("upper", upper_val, allocator); + + rapidjson::Value count_val(count); + bucket.AddMember("count", count_val, allocator); + + rapidjson::Value pre_sum_val(pre_sum); + bucket.AddMember("pre_sum", pre_sum_val, allocator); + + rapidjson::Value ndv_val(ndv); + bucket.AddMember("ndv", ndv_val, allocator); + + bucket_arr.PushBack(bucket, allocator); + + return bucket_arr.Size() > 0; +} + +template <typename T> +bool value_to_bucket(std::vector<Bucket<T>>& buckets, T v, size_t num_per_bucket) { + if (buckets.empty()) { + Bucket<T> bucket(v, 0); + buckets.emplace_back(bucket); + } else { + Bucket<T>* bucket = &buckets.back(); + T upper = bucket->upper; + if (upper == v) { + bucket->count++; + } else if (bucket->count < num_per_bucket) { + bucket->count++; + bucket->ndv++; + bucket->upper = v; + } else { + uint64_t pre_sum = bucket->pre_sum + bucket->count; + Bucket<T> new_bucket(v, pre_sum); + buckets.emplace_back(new_bucket); + } + } + + return buckets.size() > 0; +} + +template <typename T> +bool value_to_string(std::stringstream& ss, T input, const DataTypePtr& data_type) { + switch (data_type->get_type_id()) { + case TypeIndex::Int8: { + ss << *(reinterpret_cast<const int8_t*>(&input)); + break; + } + case TypeIndex::UInt8: { + ss << *(reinterpret_cast<const uint8_t*>(&input)); + break; + } + case TypeIndex::Int16: { + ss << *(reinterpret_cast<const int16_t*>(&input)); + break; + } + case TypeIndex::UInt16: { + ss << *(reinterpret_cast<const uint16_t*>(&input)); + break; + } + case TypeIndex::Int32: { + ss << *(reinterpret_cast<const int32_t*>(&input)); + break; + } + case TypeIndex::UInt32: { + ss << *(reinterpret_cast<const uint32_t*>(&input)); + break; + } + case TypeIndex::Int64: { + ss << *(reinterpret_cast<const int64_t*>(&input)); + break; + } + case TypeIndex::UInt64: { + ss << *(reinterpret_cast<const uint64_t*>(&input)); + break; + } + case TypeIndex::Int128: { + ss << fmt::format("{}", *(reinterpret_cast<const __int128_t*>(&input))); + break; + } + case TypeIndex::UInt128: { + ss << fmt::format("{}", *(reinterpret_cast<const __uint128_t*>(&input))); + break; + } + case TypeIndex::Float32: { + ss << *(reinterpret_cast<const float*>(&input)); + break; + } + case TypeIndex::Float64: { + ss << *(reinterpret_cast<const double_t*>(&input)); + break; + } + case TypeIndex::String: { + fmt::memory_buffer buffer; + fmt::format_to(buffer, "{}", input); + ss << std::string(buffer.data(), buffer.size()); + break; + } + case TypeIndex::Date: + case TypeIndex::DateTime: { + auto value = *(reinterpret_cast<const int64_t*>(&input)); + auto date_value = (VecDateTimeValue*)&value; + ss << *date_value; + break; + } + case TypeIndex::DateV2: { + auto* value = (DateV2Value<DateV2ValueType>*)(&input); + ss << *value; + break; + } + case TypeIndex::DateTimeV2: { + auto* value = (DateV2Value<DateTimeV2ValueType>*)(&input); + ss << *value; + break; + } + case TypeIndex::Decimal32: { + auto scale = get_decimal_scale(*data_type); + write_text<int32_t>(*(reinterpret_cast<const int32_t*>(&input)), scale, ss); + break; + } + case TypeIndex::Decimal64: { + auto scale = get_decimal_scale(*data_type); + write_text<int64_t>(*(reinterpret_cast<const int64_t*>(&input)), scale, ss); Review Comment: warning: use of undeclared identifier 'write_text' [clang-diagnostic-error] ```cpp write_text<int64_t>(*(reinterpret_cast<const int64_t*>(&input)), scale, ss); ^ ``` ########## be/src/vec/utils/histogram_helpers.hpp: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <rapidjson/document.h> +#include <rapidjson/prettywriter.h> +#include <rapidjson/stringbuffer.h> + +#include <boost/dynamic_bitset.hpp> + +#include "vec/data_types/data_type_decimal.h" + +namespace doris::vectorized { + +template <typename T> +struct Bucket { +public: + Bucket() = default; + Bucket(T value, size_t pre_sum) + : lower(value), upper(value), count(1), pre_sum(pre_sum), ndv(1) {} + Bucket(T lower, T upper, size_t count, size_t pre_sum, size_t ndv) + : lower(lower), upper(upper), count(count), pre_sum(pre_sum), ndv(ndv) {} + + T lower; + T upper; + uint64_t count; + uint64_t pre_sum; + uint64_t ndv; +}; + +template <typename T> +bool build_bucket_from_data(std::vector<Bucket<T>>& buckets, + const std::map<T, uint64_t>& ordered_map, double sample_rate, + uint32_t max_bucket_num) { + if (ordered_map.size() == 0) { + return false; + } + + uint64_t element_number = 0; + for (auto it : ordered_map) { + element_number += it.second; + } + + auto sample_number = (uint64_t)std::ceil(element_number * sample_rate); + auto num_per_bucket = (uint64_t)std::ceil((Float64)sample_number / max_bucket_num); + + LOG(INFO) << fmt::format( + "histogram bucket info: element number {}, sample number:{}, num per bucket:{}", + element_number, sample_number, num_per_bucket); + + if (sample_rate == 1) { + for (auto it : ordered_map) { + for (auto i = it.second; i > 0; --i) { + auto v = it.first; + value_to_bucket(buckets, v, num_per_bucket); + } + } + return true; + } + + // if sampling is required (0<sample_rate<1), + // we need to build the sampled data index + boost::dynamic_bitset<> sample_index(element_number); + + // use a same seed value so that we get + // same result each time we run this function + srand(element_number * sample_rate * max_bucket_num); + + while (sample_index.count() < sample_number) { + uint64_t num = (rand() % (element_number)); + sample_index[num] = true; + } + + uint64_t element_cnt = 0; + uint64_t sample_cnt = 0; + bool break_flag = false; + + for (auto it : ordered_map) { + if (break_flag) { + break; + } + for (auto i = it.second; i > 0; --i) { + if (sample_cnt >= sample_number) { + break_flag = true; + break; + } + if (sample_index[element_cnt]) { + sample_cnt += 1; + auto v = it.first; + value_to_bucket(buckets, v, num_per_bucket); + } + element_cnt += 1; + } + } + + return true; +} + +bool inline bucket_to_json(rapidjson::Document::AllocatorType& allocator, + rapidjson::Value& bucket_arr, std::string lower, std::string upper, + int64 count, int64 pre_sum, int64 ndv) { + rapidjson::Value bucket(rapidjson::kObjectType); + + rapidjson::Value lower_val(lower.c_str(), allocator); + bucket.AddMember("lower", lower_val, allocator); + + rapidjson::Value upper_val(upper.c_str(), allocator); + bucket.AddMember("upper", upper_val, allocator); + + rapidjson::Value count_val(count); + bucket.AddMember("count", count_val, allocator); + + rapidjson::Value pre_sum_val(pre_sum); + bucket.AddMember("pre_sum", pre_sum_val, allocator); + + rapidjson::Value ndv_val(ndv); + bucket.AddMember("ndv", ndv_val, allocator); + + bucket_arr.PushBack(bucket, allocator); + + return bucket_arr.Size() > 0; +} + +template <typename T> +bool value_to_bucket(std::vector<Bucket<T>>& buckets, T v, size_t num_per_bucket) { + if (buckets.empty()) { + Bucket<T> bucket(v, 0); + buckets.emplace_back(bucket); + } else { + Bucket<T>* bucket = &buckets.back(); + T upper = bucket->upper; + if (upper == v) { + bucket->count++; + } else if (bucket->count < num_per_bucket) { + bucket->count++; + bucket->ndv++; + bucket->upper = v; + } else { + uint64_t pre_sum = bucket->pre_sum + bucket->count; + Bucket<T> new_bucket(v, pre_sum); + buckets.emplace_back(new_bucket); + } + } + + return buckets.size() > 0; +} + +template <typename T> +bool value_to_string(std::stringstream& ss, T input, const DataTypePtr& data_type) { + switch (data_type->get_type_id()) { + case TypeIndex::Int8: { + ss << *(reinterpret_cast<const int8_t*>(&input)); + break; + } + case TypeIndex::UInt8: { + ss << *(reinterpret_cast<const uint8_t*>(&input)); + break; + } + case TypeIndex::Int16: { + ss << *(reinterpret_cast<const int16_t*>(&input)); + break; + } + case TypeIndex::UInt16: { + ss << *(reinterpret_cast<const uint16_t*>(&input)); + break; + } + case TypeIndex::Int32: { + ss << *(reinterpret_cast<const int32_t*>(&input)); + break; + } + case TypeIndex::UInt32: { + ss << *(reinterpret_cast<const uint32_t*>(&input)); + break; + } + case TypeIndex::Int64: { + ss << *(reinterpret_cast<const int64_t*>(&input)); + break; + } + case TypeIndex::UInt64: { + ss << *(reinterpret_cast<const uint64_t*>(&input)); + break; + } + case TypeIndex::Int128: { + ss << fmt::format("{}", *(reinterpret_cast<const __int128_t*>(&input))); + break; + } + case TypeIndex::UInt128: { + ss << fmt::format("{}", *(reinterpret_cast<const __uint128_t*>(&input))); + break; + } + case TypeIndex::Float32: { + ss << *(reinterpret_cast<const float*>(&input)); + break; + } + case TypeIndex::Float64: { + ss << *(reinterpret_cast<const double_t*>(&input)); + break; + } + case TypeIndex::String: { + fmt::memory_buffer buffer; + fmt::format_to(buffer, "{}", input); + ss << std::string(buffer.data(), buffer.size()); + break; + } + case TypeIndex::Date: + case TypeIndex::DateTime: { + auto value = *(reinterpret_cast<const int64_t*>(&input)); + auto date_value = (VecDateTimeValue*)&value; + ss << *date_value; + break; + } + case TypeIndex::DateV2: { + auto* value = (DateV2Value<DateV2ValueType>*)(&input); + ss << *value; + break; + } + case TypeIndex::DateTimeV2: { + auto* value = (DateV2Value<DateTimeV2ValueType>*)(&input); + ss << *value; + break; + } + case TypeIndex::Decimal32: { + auto scale = get_decimal_scale(*data_type); + write_text<int32_t>(*(reinterpret_cast<const int32_t*>(&input)), scale, ss); + break; + } + case TypeIndex::Decimal64: { + auto scale = get_decimal_scale(*data_type); + write_text<int64_t>(*(reinterpret_cast<const int64_t*>(&input)), scale, ss); + break; + } + case TypeIndex::Decimal128: + case TypeIndex::Decimal128I: { + auto scale = get_decimal_scale(*data_type); + write_text<int128_t>(*(reinterpret_cast<const int128_t*>(&input)), scale, ss); Review Comment: warning: use of undeclared identifier 'write_text' [clang-diagnostic-error] ```cpp write_text<int128_t>(*(reinterpret_cast<const int128_t*>(&input)), scale, ss); ^ ``` ########## be/test/vec/aggregate_functions/agg_histogram_test.cpp: ########## @@ -80,8 +116,8 @@ class VAggHistogramTest : public testing::Test { std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]); AggregateDataPtr place = memory.get(); agg_function->create(place); - - agg_histogram_add_elements<DataType>(agg_function, place, input_nums); + agg_histogram_add_elements<DataType>(agg_function, place, input_rows, sample_rate, + max_bucket_num); ColumnString buf; Review Comment: warning: calling a private constructor of class 'doris::vectorized::ColumnString' [clang-diagnostic-error] ```cpp ColumnString buf; ^ ``` **be/src/vec/columns/column_string.h:65:** declared private here ```cpp ColumnString() = default; ^ ``` ########## be/src/vec/utils/histogram_helpers.hpp: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <rapidjson/document.h> +#include <rapidjson/prettywriter.h> +#include <rapidjson/stringbuffer.h> + +#include <boost/dynamic_bitset.hpp> + +#include "vec/data_types/data_type_decimal.h" + +namespace doris::vectorized { + +template <typename T> +struct Bucket { +public: + Bucket() = default; + Bucket(T value, size_t pre_sum) + : lower(value), upper(value), count(1), pre_sum(pre_sum), ndv(1) {} + Bucket(T lower, T upper, size_t count, size_t pre_sum, size_t ndv) + : lower(lower), upper(upper), count(count), pre_sum(pre_sum), ndv(ndv) {} + + T lower; + T upper; + uint64_t count; + uint64_t pre_sum; + uint64_t ndv; +}; + +template <typename T> +bool build_bucket_from_data(std::vector<Bucket<T>>& buckets, + const std::map<T, uint64_t>& ordered_map, double sample_rate, + uint32_t max_bucket_num) { + if (ordered_map.size() == 0) { + return false; + } + + uint64_t element_number = 0; + for (auto it : ordered_map) { + element_number += it.second; + } + + auto sample_number = (uint64_t)std::ceil(element_number * sample_rate); + auto num_per_bucket = (uint64_t)std::ceil((Float64)sample_number / max_bucket_num); + + LOG(INFO) << fmt::format( + "histogram bucket info: element number {}, sample number:{}, num per bucket:{}", + element_number, sample_number, num_per_bucket); + + if (sample_rate == 1) { + for (auto it : ordered_map) { + for (auto i = it.second; i > 0; --i) { + auto v = it.first; + value_to_bucket(buckets, v, num_per_bucket); + } + } + return true; + } + + // if sampling is required (0<sample_rate<1), + // we need to build the sampled data index + boost::dynamic_bitset<> sample_index(element_number); + + // use a same seed value so that we get + // same result each time we run this function + srand(element_number * sample_rate * max_bucket_num); + + while (sample_index.count() < sample_number) { + uint64_t num = (rand() % (element_number)); + sample_index[num] = true; + } + + uint64_t element_cnt = 0; + uint64_t sample_cnt = 0; + bool break_flag = false; + + for (auto it : ordered_map) { + if (break_flag) { + break; + } + for (auto i = it.second; i > 0; --i) { + if (sample_cnt >= sample_number) { + break_flag = true; + break; + } + if (sample_index[element_cnt]) { + sample_cnt += 1; + auto v = it.first; + value_to_bucket(buckets, v, num_per_bucket); + } + element_cnt += 1; + } + } + + return true; +} + +bool inline bucket_to_json(rapidjson::Document::AllocatorType& allocator, + rapidjson::Value& bucket_arr, std::string lower, std::string upper, + int64 count, int64 pre_sum, int64 ndv) { + rapidjson::Value bucket(rapidjson::kObjectType); + + rapidjson::Value lower_val(lower.c_str(), allocator); + bucket.AddMember("lower", lower_val, allocator); + + rapidjson::Value upper_val(upper.c_str(), allocator); + bucket.AddMember("upper", upper_val, allocator); + + rapidjson::Value count_val(count); + bucket.AddMember("count", count_val, allocator); + + rapidjson::Value pre_sum_val(pre_sum); + bucket.AddMember("pre_sum", pre_sum_val, allocator); + + rapidjson::Value ndv_val(ndv); + bucket.AddMember("ndv", ndv_val, allocator); + + bucket_arr.PushBack(bucket, allocator); + + return bucket_arr.Size() > 0; +} + +template <typename T> +bool value_to_bucket(std::vector<Bucket<T>>& buckets, T v, size_t num_per_bucket) { + if (buckets.empty()) { + Bucket<T> bucket(v, 0); + buckets.emplace_back(bucket); + } else { + Bucket<T>* bucket = &buckets.back(); + T upper = bucket->upper; + if (upper == v) { + bucket->count++; + } else if (bucket->count < num_per_bucket) { + bucket->count++; + bucket->ndv++; + bucket->upper = v; + } else { + uint64_t pre_sum = bucket->pre_sum + bucket->count; + Bucket<T> new_bucket(v, pre_sum); + buckets.emplace_back(new_bucket); + } + } + + return buckets.size() > 0; +} + +template <typename T> +bool value_to_string(std::stringstream& ss, T input, const DataTypePtr& data_type) { + switch (data_type->get_type_id()) { + case TypeIndex::Int8: { + ss << *(reinterpret_cast<const int8_t*>(&input)); + break; + } + case TypeIndex::UInt8: { + ss << *(reinterpret_cast<const uint8_t*>(&input)); + break; + } + case TypeIndex::Int16: { + ss << *(reinterpret_cast<const int16_t*>(&input)); + break; + } + case TypeIndex::UInt16: { + ss << *(reinterpret_cast<const uint16_t*>(&input)); + break; + } + case TypeIndex::Int32: { + ss << *(reinterpret_cast<const int32_t*>(&input)); + break; + } + case TypeIndex::UInt32: { + ss << *(reinterpret_cast<const uint32_t*>(&input)); + break; + } + case TypeIndex::Int64: { + ss << *(reinterpret_cast<const int64_t*>(&input)); + break; + } + case TypeIndex::UInt64: { + ss << *(reinterpret_cast<const uint64_t*>(&input)); + break; + } + case TypeIndex::Int128: { + ss << fmt::format("{}", *(reinterpret_cast<const __int128_t*>(&input))); + break; + } + case TypeIndex::UInt128: { + ss << fmt::format("{}", *(reinterpret_cast<const __uint128_t*>(&input))); + break; + } + case TypeIndex::Float32: { + ss << *(reinterpret_cast<const float*>(&input)); + break; + } + case TypeIndex::Float64: { + ss << *(reinterpret_cast<const double_t*>(&input)); + break; + } + case TypeIndex::String: { + fmt::memory_buffer buffer; + fmt::format_to(buffer, "{}", input); + ss << std::string(buffer.data(), buffer.size()); + break; + } + case TypeIndex::Date: + case TypeIndex::DateTime: { + auto value = *(reinterpret_cast<const int64_t*>(&input)); + auto date_value = (VecDateTimeValue*)&value; + ss << *date_value; + break; + } + case TypeIndex::DateV2: { + auto* value = (DateV2Value<DateV2ValueType>*)(&input); + ss << *value; + break; + } + case TypeIndex::DateTimeV2: { + auto* value = (DateV2Value<DateTimeV2ValueType>*)(&input); + ss << *value; + break; + } + case TypeIndex::Decimal32: { + auto scale = get_decimal_scale(*data_type); + write_text<int32_t>(*(reinterpret_cast<const int32_t*>(&input)), scale, ss); Review Comment: warning: use of undeclared identifier 'write_text' [clang-diagnostic-error] ```cpp write_text<int32_t>(*(reinterpret_cast<const int32_t*>(&input)), scale, ss); ^ ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
