zclllyybb commented on code in PR #56648: URL: https://github.com/apache/doris/pull/56648#discussion_r2424206597
########## be/src/vec/utils/varbinaryop_subbinary.h: ########## @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "vec/columns/column_const.h" +#include "vec/columns/column_varbinary.h" +#include "vec/core/block.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_varbinary.h" + +namespace doris::vectorized { +#include "common/compile_check_begin.h" + +struct SubBinaryUtil { + static void sub_binary_execute(Block& block, const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) { + DCHECK_EQ(arguments.size(), 3); + auto res = ColumnVarbinary::create(); + + bool col_const[3]; + ColumnPtr argument_columns[3]; + for (int i = 0; i < 3; ++i) { + std::tie(argument_columns[i], col_const[i]) = + unpack_if_const(block.get_by_position(arguments[i]).column); + } + + const auto* specific_binary_column = + assert_cast<const ColumnVarbinary*>(argument_columns[0].get()); + const auto* specific_start_column = + assert_cast<const ColumnInt32*>(argument_columns[1].get()); + const auto* specific_len_column = + assert_cast<const ColumnInt32*>(argument_columns[2].get()); + + std::visit( + [&](auto binary_const, auto start_const, auto len_const) { + vectors<binary_const, start_const, len_const>( + specific_binary_column, specific_start_column, specific_len_column, + res.get(), input_rows_count); + }, + vectorized::make_bool_variant(col_const[0]), + vectorized::make_bool_variant(col_const[1]), + vectorized::make_bool_variant(col_const[2])); + block.get_by_position(result).column = std::move(res); + } + +private: + template <bool binary_const, bool start_const, bool len_const> + static void vectors(const ColumnVarbinary* binarys, const ColumnInt32* start, + const ColumnInt32* len, ColumnVarbinary* res, size_t size) { + if constexpr (start_const && len_const) { + if (start->get_data()[0] == 0 || len->get_data()[0] <= 0) { Review Comment: 这为啥要搞个特判路径出来？这种情况结果没啥意义，所以专门加速也没啥意义 ########## be/src/vec/functions/function_varbinary.cpp: ########## @@ -143,11 +149,181 @@ class FunctionFromBinary : public IFunction { } }; +struct NameVarbinaryLength { + static constexpr auto name = "length"; +}; + +struct VarbinaryLengthImpl { + using ReturnType = DataTypeInt32; + using ReturnColumnType = ColumnInt32; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static DataTypes get_variadic_argument_types() { + return {std::make_shared<DataTypeVarbinary>()}; + } + + static Status vector(const PaddedPODArray<doris::StringView>& data, + PaddedPODArray<Int32>& res) { + size_t rows_count = data.size(); + res.resize(rows_count); + for (size_t i = 0; i < rows_count; ++i) { + res[i] = data[i].size(); + } + return Status::OK(); + } +}; + +using FunctionBinaryLength = FunctionUnaryToType<VarbinaryLengthImpl, NameVarbinaryLength>; + +struct ToBase64BinaryImpl { + static constexpr auto name = "to_base64_binary"; + using ReturnType = DataTypeString; + using ColumnType = ColumnString; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static Status vector(const PaddedPODArray<doris::StringView>& data, + ColumnString::Chars& dst_data, ColumnString::Offsets& dst_offsets) { + auto rows_count = data.size(); + dst_offsets.resize(rows_count); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + auto binary = data[i]; + auto binlen = binary.size(); + + if (binlen == 0) { + StringOP::push_empty_string(i, dst_data, dst_offsets); + continue; + } + + char* dst = nullptr; + auto cipher_len = 4 * ((binlen + 2) / 3); + if (cipher_len <= stack_buf.size()) { + dst = stack_buf.data(); + } else { + heap_buf.resize(cipher_len); + dst = heap_buf.data(); + } + + auto outlen = + doris::base64_encode(reinterpret_cast<const unsigned char*>(binary.data()), + binlen, reinterpret_cast<unsigned char*>(dst)); + + StringOP::push_value_string(std::string_view(dst, outlen), i, dst_data, dst_offsets); + } + + return Status::OK(); + } +}; + +using FunctionToBase64Binary = FunctionStringEncode<ToBase64BinaryImpl, false>; + +struct FromBase64BinaryImpl { + static constexpr auto name = "from_base64_binary"; + using ReturnType = DataTypeVarbinary; + using ColumnType = ColumnVarbinary; + + static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, + ColumnVarbinary* res, NullMap& null_map) { + auto rows_count = offsets.size(); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]); + ColumnString::Offset slen = offsets[i] - offsets[i - 1]; + + if (slen == 0) { + res->insert_default(); + continue; + } + + UInt32 cipher_len = slen; + char* dst = nullptr; + if (cipher_len <= stack_buf.size()) { + dst = stack_buf.data(); + } else { + heap_buf.resize(cipher_len); + dst = heap_buf.data(); + } + + auto outlen = doris::base64_decode(source, slen, dst); + + if (outlen < 0) { + null_map[i] = 1; + res->insert_default(); + } else { + res->insert_data(dst, outlen); + } + } + + return Status::OK(); + } +}; + +using FunctionFromBase64Binary = FunctionStringOperateToNullType<FromBase64BinaryImpl>; + +struct SubBinary3Impl { + static DataTypes get_variadic_argument_types() { + return DataTypes {std::make_shared<DataTypeVarbinary>(), std::make_shared<DataTypeInt32>(), + std::make_shared<DataTypeInt32>()}; + } + + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) { + SubBinaryUtil::sub_binary_execute(block, arguments, result, input_rows_count); + return Status::OK(); + } +}; + +struct SubBinary2Impl { + static DataTypes get_variadic_argument_types() { + return DataTypes {std::make_shared<DataTypeVarbinary>(), std::make_shared<DataTypeInt32>()}; + } + + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) { + auto col_len = ColumnInt32::create(input_rows_count); + auto& strlen_data = col_len->get_data(); + + ColumnPtr binary_col; + bool binary_const; + std::tie(binary_col, binary_const) = + unpack_if_const(block.get_by_position(arguments[0]).column); + + const auto& binarys = assert_cast<const ColumnVarbinary*>(binary_col.get()); + + if (binary_const) { + std::fill(strlen_data.begin(), strlen_data.end(), binarys->get_data()[0].size()); + } else { + for (int i = 0; i < input_rows_count; ++i) { + strlen_data[i] = binarys->get_data()[i].size(); + } + } + + // we complete the column2(strlen) with the default value - each row's strlen. + block.insert({std::move(col_len), std::make_shared<DataTypeInt32>(), "strlen"}); Review Comment: 或者你可以考虑直接在FE端做一个函数改写也可以。加个length函数不会拖慢性能的 ########## be/src/vec/utils/varbinaryop_subbinary.h: ########## @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "vec/columns/column_const.h" +#include "vec/columns/column_varbinary.h" +#include "vec/core/block.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_varbinary.h" + +namespace doris::vectorized { +#include "common/compile_check_begin.h" + +struct SubBinaryUtil { + static void sub_binary_execute(Block& block, const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) { + DCHECK_EQ(arguments.size(), 3); + auto res = ColumnVarbinary::create(); + + bool col_const[3]; + ColumnPtr argument_columns[3]; + for (int i = 0; i < 3; ++i) { + std::tie(argument_columns[i], col_const[i]) = + unpack_if_const(block.get_by_position(arguments[i]).column); + } + + const auto* specific_binary_column = + assert_cast<const ColumnVarbinary*>(argument_columns[0].get()); + const auto* specific_start_column = + assert_cast<const ColumnInt32*>(argument_columns[1].get()); + const auto* specific_len_column = + assert_cast<const ColumnInt32*>(argument_columns[2].get()); + + std::visit( + [&](auto binary_const, auto start_const, auto len_const) { + vectors<binary_const, start_const, len_const>( + specific_binary_column, specific_start_column, specific_len_column, + res.get(), input_rows_count); + }, + vectorized::make_bool_variant(col_const[0]), + vectorized::make_bool_variant(col_const[1]), + vectorized::make_bool_variant(col_const[2])); + block.get_by_position(result).column = std::move(res); + } + +private: + template <bool binary_const, bool start_const, bool len_const> + static void vectors(const ColumnVarbinary* binarys, const ColumnInt32* start, + const ColumnInt32* len, ColumnVarbinary* res, size_t size) { + if constexpr (start_const && len_const) { Review Comment: 这个情况下应该比较常见，此时可以给res做一下reserve来提升性能。 ########## be/src/vec/functions/function_varbinary.cpp: ########## @@ -143,11 +149,181 @@ class FunctionFromBinary : public IFunction { } }; +struct NameVarbinaryLength { + static constexpr auto name = "length"; +}; + +struct VarbinaryLengthImpl { + using ReturnType = DataTypeInt32; + using ReturnColumnType = ColumnInt32; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static DataTypes get_variadic_argument_types() { + return {std::make_shared<DataTypeVarbinary>()}; + } + + static Status vector(const PaddedPODArray<doris::StringView>& data, + PaddedPODArray<Int32>& res) { + size_t rows_count = data.size(); + res.resize(rows_count); + for (size_t i = 0; i < rows_count; ++i) { + res[i] = data[i].size(); + } + return Status::OK(); + } +}; + +using FunctionBinaryLength = FunctionUnaryToType<VarbinaryLengthImpl, NameVarbinaryLength>; + +struct ToBase64BinaryImpl { + static constexpr auto name = "to_base64_binary"; + using ReturnType = DataTypeString; + using ColumnType = ColumnString; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static Status vector(const PaddedPODArray<doris::StringView>& data, + ColumnString::Chars& dst_data, ColumnString::Offsets& dst_offsets) { + auto rows_count = data.size(); + dst_offsets.resize(rows_count); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + auto binary = data[i]; + auto binlen = binary.size(); + + if (binlen == 0) { + StringOP::push_empty_string(i, dst_data, dst_offsets); + continue; + } + + char* dst = nullptr; + auto cipher_len = 4 * ((binlen + 2) / 3); + if (cipher_len <= stack_buf.size()) { + dst = stack_buf.data(); + } else { + heap_buf.resize(cipher_len); + dst = heap_buf.data(); + } + + auto outlen = + doris::base64_encode(reinterpret_cast<const unsigned char*>(binary.data()), + binlen, reinterpret_cast<unsigned char*>(dst)); + + StringOP::push_value_string(std::string_view(dst, outlen), i, dst_data, dst_offsets); + } + + return Status::OK(); + } +}; + +using FunctionToBase64Binary = FunctionStringEncode<ToBase64BinaryImpl, false>; + +struct FromBase64BinaryImpl { + static constexpr auto name = "from_base64_binary"; + using ReturnType = DataTypeVarbinary; + using ColumnType = ColumnVarbinary; + + static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, + ColumnVarbinary* res, NullMap& null_map) { + auto rows_count = offsets.size(); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]); + ColumnString::Offset slen = offsets[i] - offsets[i - 1]; + + if (slen == 0) { + res->insert_default(); + continue; + } + + UInt32 cipher_len = slen; + char* dst = nullptr; Review Comment: base64解码后的长度(3/4输入-填充)是能计算出来的，或者最起码能计算出来上限。所以可以避免拷贝dst，直接resize res然后写到里面，最后shrink下应该就行了 ########## regression-test/suites/query_p0/sql_functions/binary_functions/test_binary_function.groovy: ########## @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_binary_function", "p0,external,mysql,external_docker,external_docker_mysql") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar"; + String catalog_name = "mysql_varbinary_catalog"; + String ex_db_name = "doris_test"; + String mysql_port = context.config.otherConfigs.get("mysql_57_port"); + String test_table = "binary_test"; + + + + sql """drop catalog if exists ${catalog_name}""" + + sql """create catalog if not exists ${catalog_name} properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver" + );""" + + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false") { + try_sql """DROP TABLE IF EXISTS ${test_table}""" + + sql """CREATE TABLE ${test_table} ( + id int, + vb varbinary(100), + vc VARCHAR(100) + )""" + + sql """INSERT INTO ${test_table} VALUES + (1, 'hello world', 'hello world'), + (2, '', ''), + (3, 'special chars: !@#%', 'special chars: !@#%'), + (4, '__123hehe1', '__123hehe1'), + (5, 'ABB', 'ABB'), + (6, '5ZWK5ZOI5ZOI5ZOI8J+YhCDjgILigJTigJQh', '5ZWK5ZOI5ZOI5ZOI8J+YhCDjgILigJTigJQh'), + (7, 'SEVMTE8sIV4l', 'SEVMTE8sIV4l') + """ + } + + sql """switch ${catalog_name}""" + sql """use ${ex_db_name}""" + + def length_result = sql """select id, length(vb), length(vc) from ${test_table} order by id""" + for (int i = 0; i < length_result.size(); i++) { + assertTrue(length_result[i][1] == length_result[i][2], + "length mismatch for row ${length_result[i][0]}: VarBinary=${length_result[i][1]}, VARCHAR=${length_result[i][2]}") + } + + def from_base64_result = sql """select id, from_binary(from_base64_binary(vc)), hex(from_base64(vc)) from ${test_table} order by id""" + for (int i = 0; i < from_base64_result.size(); i++) { + def bin = from_base64_result[i][1] + def str = from_base64_result[i][2] + assertTrue(bin == str, + "from_base64 mismatch for row ${from_base64_result[i][0]}: VarBinary=${bin}, VARCHAR=${str}") + } + + def to_base64_result = sql """select id, to_base64_binary(vb), to_base64(vc) from ${test_table} order by id""" + for (int i = 0; i < to_base64_result.size(); i++) { + assertTrue(to_base64_result[i][1] == to_base64_result[i][2], + "to_base64 mismatch for row ${to_base64_result[i][0]}: VarBinary=${to_base64_result[i][1]}, VARCHAR=${to_base64_result[i][2]}") + } + + def sub_binary_3args_result = sql """select id, from_binary(sub_binary(vb, 1, 5)), hex(substr(vc, 1, 5)) from ${test_table} order by id""" + for (int i = 0; i < sub_binary_3args_result.size(); i++) { + def bin = sub_binary_3args_result[i][1] + def str = sub_binary_3args_result[i][2] + assertTrue(bin == str, + "sub_binary_3args mismatch for row ${sub_binary_3args_result[i][0]}: VarBinary=${bin}, VARCHAR=${str}") + } + + def sub_binary_3args_result_2 = sql """select id, from_binary(sub_binary(vb, -1, 5)), hex(substr(vc, -1, 5)) from ${test_table} order by id""" + for (int i = 0; i < sub_binary_3args_result_2.size(); i++) { + def bin = sub_binary_3args_result_2[i][1] + def str = sub_binary_3args_result_2[i][2] + assertTrue(bin == str, + "sub_binary_3args_2 mismatch for row ${sub_binary_3args_result_2[i][0]}: VarBinary=${bin}, VARCHAR=${str}") + } + + def sub_binary_2args_result = sql """select id, from_binary(sub_binary(vb, 1)), hex(substr(vc, 1)) from ${test_table} order by id""" + for (int i = 0; i < sub_binary_2args_result.size(); i++) { + def bin = sub_binary_2args_result[i][1] + def str = sub_binary_2args_result[i][2] + assertTrue(bin == str, + "sub_binary_2args mismatch for row ${sub_binary_2args_result[i][0]}: VarBinary=${bin}, VARCHAR=${str}") + } + + def sub_binary_2args_result_2 = sql """select id, from_binary(sub_binary(vb, -1)), hex(substr(vc, -1)) from ${test_table} order by id""" + for (int i = 0; i < sub_binary_2args_result_2.size(); i++) { + def bin = sub_binary_2args_result_2[i][1] + def str = sub_binary_2args_result_2[i][2] + assertTrue(bin == str, + "sub_binary_2args_2 mismatch for row ${sub_binary_2args_result_2[i][0]}: VarBinary=${bin}, VARCHAR=${str}") + } + + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false") { + try_sql """DROP TABLE IF EXISTS ${test_table}""" + } + + sql """drop catalog if exists ${catalog_name}""" Review Comment: 最终一般不要drop table和catalog了，除非要当前case 复用。这样方便debug。 ########## be/src/vec/functions/function_varbinary.cpp: ########## @@ -143,11 +149,181 @@ class FunctionFromBinary : public IFunction { } }; +struct NameVarbinaryLength { + static constexpr auto name = "length"; +}; + +struct VarbinaryLengthImpl { + using ReturnType = DataTypeInt32; + using ReturnColumnType = ColumnInt32; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static DataTypes get_variadic_argument_types() { + return {std::make_shared<DataTypeVarbinary>()}; + } + + static Status vector(const PaddedPODArray<doris::StringView>& data, + PaddedPODArray<Int32>& res) { + size_t rows_count = data.size(); + res.resize(rows_count); + for (size_t i = 0; i < rows_count; ++i) { + res[i] = data[i].size(); + } + return Status::OK(); + } +}; + +using FunctionBinaryLength = FunctionUnaryToType<VarbinaryLengthImpl, NameVarbinaryLength>; + +struct ToBase64BinaryImpl { + static constexpr auto name = "to_base64_binary"; + using ReturnType = DataTypeString; + using ColumnType = ColumnString; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static Status vector(const PaddedPODArray<doris::StringView>& data, + ColumnString::Chars& dst_data, ColumnString::Offsets& dst_offsets) { + auto rows_count = data.size(); + dst_offsets.resize(rows_count); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + auto binary = data[i]; + auto binlen = binary.size(); + + if (binlen == 0) { Review Comment: 这种都可以顺手加一个unlikely ########## be/src/vec/functions/function_varbinary.cpp: ########## @@ -143,11 +149,181 @@ class FunctionFromBinary : public IFunction { } }; +struct NameVarbinaryLength { + static constexpr auto name = "length"; +}; + +struct VarbinaryLengthImpl { + using ReturnType = DataTypeInt32; + using ReturnColumnType = ColumnInt32; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static DataTypes get_variadic_argument_types() { + return {std::make_shared<DataTypeVarbinary>()}; + } + + static Status vector(const PaddedPODArray<doris::StringView>& data, + PaddedPODArray<Int32>& res) { + size_t rows_count = data.size(); + res.resize(rows_count); + for (size_t i = 0; i < rows_count; ++i) { + res[i] = data[i].size(); + } + return Status::OK(); + } +}; + +using FunctionBinaryLength = FunctionUnaryToType<VarbinaryLengthImpl, NameVarbinaryLength>; + +struct ToBase64BinaryImpl { + static constexpr auto name = "to_base64_binary"; + using ReturnType = DataTypeString; + using ColumnType = ColumnString; + static constexpr auto PrimitiveTypeImpl = PrimitiveType::TYPE_VARBINARY; + + static Status vector(const PaddedPODArray<doris::StringView>& data, + ColumnString::Chars& dst_data, ColumnString::Offsets& dst_offsets) { + auto rows_count = data.size(); + dst_offsets.resize(rows_count); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + auto binary = data[i]; + auto binlen = binary.size(); + + if (binlen == 0) { + StringOP::push_empty_string(i, dst_data, dst_offsets); + continue; + } + + char* dst = nullptr; + auto cipher_len = 4 * ((binlen + 2) / 3); + if (cipher_len <= stack_buf.size()) { + dst = stack_buf.data(); + } else { + heap_buf.resize(cipher_len); + dst = heap_buf.data(); + } + + auto outlen = + doris::base64_encode(reinterpret_cast<const unsigned char*>(binary.data()), + binlen, reinterpret_cast<unsigned char*>(dst)); + + StringOP::push_value_string(std::string_view(dst, outlen), i, dst_data, dst_offsets); + } + + return Status::OK(); + } +}; + +using FunctionToBase64Binary = FunctionStringEncode<ToBase64BinaryImpl, false>; + +struct FromBase64BinaryImpl { + static constexpr auto name = "from_base64_binary"; + using ReturnType = DataTypeVarbinary; + using ColumnType = ColumnVarbinary; + + static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, + ColumnVarbinary* res, NullMap& null_map) { + auto rows_count = offsets.size(); + + std::array<char, string_hex::MAX_STACK_CIPHER_LEN> stack_buf; + std::vector<char> heap_buf; + for (size_t i = 0; i < rows_count; i++) { + const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]); + ColumnString::Offset slen = offsets[i] - offsets[i - 1]; + + if (slen == 0) { + res->insert_default(); + continue; + } + + UInt32 cipher_len = slen; + char* dst = nullptr; + if (cipher_len <= stack_buf.size()) { + dst = stack_buf.data(); + } else { + heap_buf.resize(cipher_len); + dst = heap_buf.data(); + } + + auto outlen = doris::base64_decode(source, slen, dst); + + if (outlen < 0) { + null_map[i] = 1; + res->insert_default(); + } else { + res->insert_data(dst, outlen); + } + } + + return Status::OK(); + } +}; + +using FunctionFromBase64Binary = FunctionStringOperateToNullType<FromBase64BinaryImpl>; + +struct SubBinary3Impl { + static DataTypes get_variadic_argument_types() { + return DataTypes {std::make_shared<DataTypeVarbinary>(), std::make_shared<DataTypeInt32>(), + std::make_shared<DataTypeInt32>()}; + } + + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) { + SubBinaryUtil::sub_binary_execute(block, arguments, result, input_rows_count); + return Status::OK(); + } +}; + +struct SubBinary2Impl { + static DataTypes get_variadic_argument_types() { + return DataTypes {std::make_shared<DataTypeVarbinary>(), std::make_shared<DataTypeInt32>()}; + } + + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) { + auto col_len = ColumnInt32::create(input_rows_count); + auto& strlen_data = col_len->get_data(); + + ColumnPtr binary_col; + bool binary_const; + std::tie(binary_col, binary_const) = + unpack_if_const(block.get_by_position(arguments[0]).column); + + const auto& binarys = assert_cast<const ColumnVarbinary*>(binary_col.get()); + + if (binary_const) { + std::fill(strlen_data.begin(), strlen_data.end(), binarys->get_data()[0].size()); + } else { + for (int i = 0; i < input_rows_count; ++i) { + strlen_data[i] = binarys->get_data()[i].size(); + } + } + + // we complete the column2(strlen) with the default value - each row's strlen. + block.insert({std::move(col_len), std::make_shared<DataTypeInt32>(), "strlen"}); Review Comment: 这个列最好执行完drop掉。我们再加一些嵌套执行这个函数的SQL测试吧，感觉不是很稳。 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
