github-actions[bot] commented on code in PR #34929: URL: https://github.com/apache/doris/pull/34929#discussion_r1613537492
########## be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp: ########## @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "viceberg_partition_writer.h" + +#include "io/file_factory.h" +#include "runtime/runtime_state.h" +#include "vec/columns/column_map.h" +#include "vec/core/materialize_block.h" +#include "vec/exec/format/table/iceberg/schema.h" +#include "vec/runtime/vorc_transformer.h" +#include "vec/runtime/vparquet_transformer.h" + +namespace doris { +namespace vectorized { + +VIcebergPartitionWriter::VIcebergPartitionWriter( + const TDataSink& t_sink, std::vector<std::string> partition_values, + const VExprContextSPtrs& output_expr_ctxs, const VExprContextSPtrs& write_output_expr_ctxs, + const std::set<size_t>& non_write_columns_indices, const iceberg::Schema& schema, + WriteInfo write_info, std::string file_name, int file_name_index, + TFileFormatType::type file_format_type, TFileCompressType::type compress_type, + const std::map<std::string, std::string>& hadoop_conf) + : _partition_values(std::move(partition_values)), + _vec_output_expr_ctxs(output_expr_ctxs), + _write_output_expr_ctxs(write_output_expr_ctxs), + _non_write_columns_indices(non_write_columns_indices), + _schema(schema), + _write_info(std::move(write_info)), + _file_name(std::move(file_name)), + _file_name_index(file_name_index), + _file_format_type(file_format_type), + _compress_type(compress_type), + _hadoop_conf(hadoop_conf) {} + +Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profile) { Review Comment: warning: function 'open' exceeds recommended size/complexity thresholds [readability-function-size] ```cpp Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profile) { ^ ``` <details> <summary>Additional context</summary> **be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp:49:** 81 lines including whitespace and comments (threshold 80) ```cpp Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profile) { ^ ``` </details> ########## be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h: ########## @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <gen_cpp/DataSinks_types.h> Review Comment: warning: 'gen_cpp/DataSinks_types.h' file not found [clang-diagnostic-error] ```cpp #include <gen_cpp/DataSinks_types.h> ^ ``` ########## be/src/vec/sink/writer/iceberg/partition_transformers.h: ########## @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/types.h" +#include "util/bit_util.h" +#include "vec/data_types/data_type_factory.hpp" +#include "vec/functions/function_string.h" + +namespace doris { + +namespace iceberg { +class Type; +class PartitionField; +}; // namespace iceberg + +namespace vectorized { + +class IColumn; +class PartitionColumnTransform; + +class PartitionColumnTransforms { +private: + PartitionColumnTransforms(); + +public: + static std::unique_ptr<PartitionColumnTransform> create( + const doris::iceberg::PartitionField& field, const TypeDescriptor& source_type); +}; + +class PartitionColumnTransform { +public: + PartitionColumnTransform() = default; + + virtual ~PartitionColumnTransform() = default; + + virtual bool preserves_non_null() const { return false; } + + virtual bool monotonic() const { return true; } + + virtual bool temporal() const { return false; } + + virtual const TypeDescriptor& get_result_type() const = 0; + + virtual bool is_void() const { return false; } + + virtual ColumnWithTypeAndName apply(Block& block, int idx) = 0; + + virtual std::string to_human_string(const TypeDescriptor& type, const std::any& value) const; +}; + +class IdentityPartitionColumnTransform : public PartitionColumnTransform { +public: + IdentityPartitionColumnTransform(const TypeDescriptor& source_type) + : _source_type(source_type) {} + + virtual const TypeDescriptor& get_result_type() const { return _source_type; } + + virtual ColumnWithTypeAndName apply(Block& block, int idx); Review Comment: warning: prefer using 'override' or (rarely) 'final' instead of 'virtual' [modernize-use-override] ```suggestion ColumnWithTypeAndName apply(Block& block, int idx) override; ``` ########## be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp: ########## @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "viceberg_partition_writer.h" + +#include "io/file_factory.h" +#include "runtime/runtime_state.h" +#include "vec/columns/column_map.h" +#include "vec/core/materialize_block.h" +#include "vec/exec/format/table/iceberg/schema.h" +#include "vec/runtime/vorc_transformer.h" +#include "vec/runtime/vparquet_transformer.h" + +namespace doris { +namespace vectorized { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::vectorized { ``` be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp:248: ```diff - } // namespace vectorized - } // namespace doris + } // namespace doris ``` ########## be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp: ########## @@ -0,0 +1,450 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "viceberg_table_writer.h" + +#include "runtime/runtime_state.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/exec/format/table/iceberg/partition_spec_parser.h" +#include "vec/exec/format/table/iceberg/schema_parser.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" +#include "vec/exprs/vslot_ref.h" +#include "vec/sink/writer/iceberg/partition_transformers.h" +#include "vec/sink/writer/iceberg/viceberg_partition_writer.h" +#include "vec/sink/writer/vhive_utils.h" + +namespace doris { +namespace vectorized { + +VIcebergTableWriter::VIcebergTableWriter(const TDataSink& t_sink, + const VExprContextSPtrs& output_expr_ctxs) + : AsyncResultWriter(output_expr_ctxs), _t_sink(t_sink) { + DCHECK(_t_sink.__isset.iceberg_table_sink); +} + +Status VIcebergTableWriter::init_properties(ObjectPool* pool) { + return Status::OK(); +} + +Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { + _state = state; + _profile = profile; + + // add all counter + _written_rows_counter = ADD_COUNTER(_profile, "WrittenRows", TUnit::UNIT); + _send_data_timer = ADD_TIMER(_profile, "SendDataTime"); + _partition_writers_dispatch_timer = + ADD_CHILD_TIMER(_profile, "PartitionsDispatchTime", "SendDataTime"); + _partition_writers_write_timer = + ADD_CHILD_TIMER(_profile, "PartitionsWriteTime", "SendDataTime"); + _partition_writers_count = ADD_COUNTER(_profile, "PartitionsWriteCount", TUnit::UNIT); + _open_timer = ADD_TIMER(_profile, "OpenTime"); + _close_timer = ADD_TIMER(_profile, "CloseTime"); + _write_file_counter = ADD_COUNTER(_profile, "WriteFileCount", TUnit::UNIT); + + SCOPED_TIMER(_open_timer); + try { + _schema = iceberg::SchemaParser::from_json(_t_sink.iceberg_table_sink.schema_json); + std::string partition_spec_json = + _t_sink.iceberg_table_sink + .partition_specs_json[_t_sink.iceberg_table_sink.partition_spec_id]; + if (!partition_spec_json.empty()) { + _partition_spec = iceberg::PartitionSpecParser::from_json(_schema, partition_spec_json); + _iceberg_partition_columns = _to_iceberg_partition_columns(); + } + } catch (doris::Exception& e) { + return e.to_status(); + } + + std::set<int> partition_idx_set; + for (const auto& iceberg_partition_column : _iceberg_partition_columns) { + partition_idx_set.insert(iceberg_partition_column.source_idx()); + } + + for (int i = 0; i < _schema->columns().size(); ++i) { + _write_output_vexpr_ctxs.emplace_back(_vec_output_expr_ctxs[i]); + } + + return Status::OK(); +} + +std::vector<VIcebergTableWriter::IcebergPartitionColumn> +VIcebergTableWriter::_to_iceberg_partition_columns() { + std::vector<IcebergPartitionColumn> partition_columns; + + std::unordered_map<int, int> id_to_column_idx; + id_to_column_idx.reserve(_schema->columns().size()); + for (int i = 0; i < _schema->columns().size(); i++) { + id_to_column_idx[_schema->columns()[i].field_id()] = i; + } + for (const auto& partition_field : _partition_spec->fields()) { + int column_idx = id_to_column_idx[partition_field.source_id()]; + std::unique_ptr<PartitionColumnTransform> partition_column_transform = + PartitionColumnTransforms::create( + partition_field, _vec_output_expr_ctxs[column_idx]->root()->type()); + partition_columns.emplace_back(partition_field, + _vec_output_expr_ctxs[column_idx]->root()->type(), + column_idx, std::move(partition_column_transform)); + } + return partition_columns; +} + +Status VIcebergTableWriter::write(vectorized::Block& block) { Review Comment: warning: function 'write' exceeds recommended size/complexity thresholds [readability-function-size] ```cpp Status VIcebergTableWriter::write(vectorized::Block& block) { ^ ``` <details> <summary>Additional context</summary> **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:107:** 136 lines including whitespace and comments (threshold 80) ```cpp Status VIcebergTableWriter::write(vectorized::Block& block) { ^ ``` </details> ########## be/test/vec/exec/format/table/iceberg/type_test.cpp: ########## @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gtest/gtest.h> + +#include "vec/exec/format/table/iceberg/types.h" + +namespace doris { +namespace iceberg { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::iceberg { ``` be/test/vec/exec/format/table/iceberg/type_test.cpp:174: ```diff - } // namespace iceberg - } // namespace doris + } // namespace doris ``` ########## be/src/vec/sink/writer/iceberg/partition_transformers.cpp: ########## @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/sink/writer/iceberg/partition_transformers.h" + +#include <any> + +#include "vec/core/types.h" +#include "vec/exec/format/table/iceberg/partition_spec.h" + +namespace doris { +namespace vectorized { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::vectorized { ``` be/src/vec/sink/writer/iceberg/partition_transformers.cpp:147: ```diff - } // namespace vectorized - } // namespace doris + } // namespace doris ``` ########## be/src/vec/sink/writer/iceberg/viceberg_table_writer.h: ########## @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <gen_cpp/DataSinks_types.h> + +#include <optional> + +#include "util/runtime_profile.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/exec/format/table/iceberg/partition_spec_parser.h" +#include "vec/exec/format/table/iceberg/schema_parser.h" +#include "vec/exprs/vexpr_fwd.h" +#include "vec/sink/writer/async_result_writer.h" +#include "vec/sink/writer/iceberg/partition_data.h" +#include "vec/sink/writer/iceberg/partition_transformers.h" + +namespace doris { + +class ObjectPool; +class RuntimeState; +struct TypeDescriptor; + +namespace vectorized { + +class IColumn; +class VIcebergPartitionWriter; +struct ColumnWithTypeAndName; + +class VIcebergTableWriter final : public AsyncResultWriter { +public: + VIcebergTableWriter(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs); + + ~VIcebergTableWriter() = default; + + Status init_properties(ObjectPool* pool); + + Status open(RuntimeState* state, RuntimeProfile* profile) override; + + Status write(vectorized::Block& block) override; + + Status close(Status) override; + +private: + class IcebergPartitionColumn { + public: + IcebergPartitionColumn(const iceberg::PartitionField& field, + const TypeDescriptor& source_type, int source_idx, + std::unique_ptr<PartitionColumnTransform> partition_column_transform) + : _field(field), + _source_type(source_type), + _source_idx(source_idx), + _partition_column_transform(std::move(partition_column_transform)) {} + + public: Review Comment: warning: redundant access specifier has the same accessibility as the previous access specifier [readability-redundant-access-specifiers] ```suggestion ``` <details> <summary>Additional context</summary> **be/src/vec/sink/writer/iceberg/viceberg_table_writer.h:61:** previously declared here ```cpp public: ^ ``` </details> ########## be/src/vec/sink/writer/iceberg/viceberg_table_writer.h: ########## @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <gen_cpp/DataSinks_types.h> Review Comment: warning: 'gen_cpp/DataSinks_types.h' file not found [clang-diagnostic-error] ```cpp #include <gen_cpp/DataSinks_types.h> ^ ``` ########## be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp: ########## @@ -0,0 +1,450 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "viceberg_table_writer.h" + +#include "runtime/runtime_state.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/exec/format/table/iceberg/partition_spec_parser.h" +#include "vec/exec/format/table/iceberg/schema_parser.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" +#include "vec/exprs/vslot_ref.h" +#include "vec/sink/writer/iceberg/partition_transformers.h" +#include "vec/sink/writer/iceberg/viceberg_partition_writer.h" +#include "vec/sink/writer/vhive_utils.h" + +namespace doris { +namespace vectorized { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::vectorized { ``` be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:448: ```diff - } // namespace vectorized - } // namespace doris + } // namespace doris ``` ########## be/test/vec/exec/format/table/iceberg/schema_test.cpp: ########## @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exec/format/table/iceberg/schema.h" + +#include <gtest/gtest.h> + +namespace doris { +namespace iceberg { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::iceberg { ``` be/test/vec/exec/format/table/iceberg/schema_test.cpp:68: ```diff - } // namespace iceberg - } // namespace doris + } // namespace doris ``` ########## be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h: ########## @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <gen_cpp/DataSinks_types.h> + +#include "io/fs/file_writer.h" +#include "vec/columns/column.h" +#include "vec/exec/format/table/iceberg/schema.h" +#include "vec/exprs/vexpr_fwd.h" +#include "vec/runtime/vfile_format_transformer.h" + +namespace doris { +namespace io { +class FileSystem; +} + +class ObjectPool; +class RuntimeState; +class RuntimeProfile; + +namespace iceberg { +class Schema; +} + +namespace vectorized { + +class Block; +class VFileFormatTransformer; + +class VIcebergPartitionWriter { +public: + struct WriteInfo { + std::string write_path; + std::string original_write_path; + std::string target_path; + TFileType::type file_type; + }; + + VIcebergPartitionWriter(const TDataSink& t_sink, std::vector<std::string> partition_values, + const VExprContextSPtrs& output_expr_ctxs, + const VExprContextSPtrs& write_output_expr_ctxs, + const std::set<size_t>& non_write_columns_indices, + const doris::iceberg::Schema& schema, WriteInfo write_info, + std::string file_name, int file_name_index, + TFileFormatType::type file_format_type, + TFileCompressType::type compress_type, + const std::map<std::string, std::string>& hadoop_conf); + + Status init_properties(ObjectPool* pool) { return Status::OK(); } + + Status open(RuntimeState* state, RuntimeProfile* profile); + + Status write(vectorized::Block& block, IColumn::Filter* filter = nullptr); + + Status close(const Status& status); + + inline const std::string& file_name() const { return _file_name; } + + inline int file_name_index() const { return _file_name_index; } + + inline size_t written_len() { return _file_format_transformer->written_len(); } + +private: + std::string _get_target_file_name(); + +private: Review Comment: warning: redundant access specifier has the same accessibility as the previous access specifier [readability-redundant-access-specifiers] ```suggestion ``` <details> <summary>Additional context</summary> **be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h:78:** previously declared here ```cpp private: ^ ``` </details> ########## be/src/vec/sink/writer/iceberg/viceberg_table_writer.h: ########## @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <gen_cpp/DataSinks_types.h> + +#include <optional> + +#include "util/runtime_profile.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/exec/format/table/iceberg/partition_spec_parser.h" +#include "vec/exec/format/table/iceberg/schema_parser.h" +#include "vec/exprs/vexpr_fwd.h" +#include "vec/sink/writer/async_result_writer.h" +#include "vec/sink/writer/iceberg/partition_data.h" +#include "vec/sink/writer/iceberg/partition_transformers.h" + +namespace doris { + +class ObjectPool; +class RuntimeState; +struct TypeDescriptor; + +namespace vectorized { + +class IColumn; +class VIcebergPartitionWriter; +struct ColumnWithTypeAndName; + +class VIcebergTableWriter final : public AsyncResultWriter { +public: + VIcebergTableWriter(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs); + + ~VIcebergTableWriter() = default; Review Comment: warning: annotate this function with 'override' or (rarely) 'final' [modernize-use-override] ```suggestion ~VIcebergTableWriter() override = default; ``` ########## be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp: ########## @@ -0,0 +1,450 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "viceberg_table_writer.h" + +#include "runtime/runtime_state.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/exec/format/table/iceberg/partition_spec_parser.h" +#include "vec/exec/format/table/iceberg/schema_parser.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" +#include "vec/exprs/vslot_ref.h" +#include "vec/sink/writer/iceberg/partition_transformers.h" +#include "vec/sink/writer/iceberg/viceberg_partition_writer.h" +#include "vec/sink/writer/vhive_utils.h" + +namespace doris { +namespace vectorized { + +VIcebergTableWriter::VIcebergTableWriter(const TDataSink& t_sink, + const VExprContextSPtrs& output_expr_ctxs) + : AsyncResultWriter(output_expr_ctxs), _t_sink(t_sink) { + DCHECK(_t_sink.__isset.iceberg_table_sink); +} + +Status VIcebergTableWriter::init_properties(ObjectPool* pool) { + return Status::OK(); +} + +Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { + _state = state; + _profile = profile; + + // add all counter + _written_rows_counter = ADD_COUNTER(_profile, "WrittenRows", TUnit::UNIT); + _send_data_timer = ADD_TIMER(_profile, "SendDataTime"); + _partition_writers_dispatch_timer = + ADD_CHILD_TIMER(_profile, "PartitionsDispatchTime", "SendDataTime"); + _partition_writers_write_timer = + ADD_CHILD_TIMER(_profile, "PartitionsWriteTime", "SendDataTime"); + _partition_writers_count = ADD_COUNTER(_profile, "PartitionsWriteCount", TUnit::UNIT); + _open_timer = ADD_TIMER(_profile, "OpenTime"); + _close_timer = ADD_TIMER(_profile, "CloseTime"); + _write_file_counter = ADD_COUNTER(_profile, "WriteFileCount", TUnit::UNIT); + + SCOPED_TIMER(_open_timer); + try { + _schema = iceberg::SchemaParser::from_json(_t_sink.iceberg_table_sink.schema_json); + std::string partition_spec_json = + _t_sink.iceberg_table_sink + .partition_specs_json[_t_sink.iceberg_table_sink.partition_spec_id]; + if (!partition_spec_json.empty()) { + _partition_spec = iceberg::PartitionSpecParser::from_json(_schema, partition_spec_json); + _iceberg_partition_columns = _to_iceberg_partition_columns(); + } + } catch (doris::Exception& e) { + return e.to_status(); + } + + std::set<int> partition_idx_set; + for (const auto& iceberg_partition_column : _iceberg_partition_columns) { + partition_idx_set.insert(iceberg_partition_column.source_idx()); + } + + for (int i = 0; i < _schema->columns().size(); ++i) { + _write_output_vexpr_ctxs.emplace_back(_vec_output_expr_ctxs[i]); + } + + return Status::OK(); +} + +std::vector<VIcebergTableWriter::IcebergPartitionColumn> +VIcebergTableWriter::_to_iceberg_partition_columns() { + std::vector<IcebergPartitionColumn> partition_columns; + + std::unordered_map<int, int> id_to_column_idx; + id_to_column_idx.reserve(_schema->columns().size()); + for (int i = 0; i < _schema->columns().size(); i++) { + id_to_column_idx[_schema->columns()[i].field_id()] = i; + } + for (const auto& partition_field : _partition_spec->fields()) { + int column_idx = id_to_column_idx[partition_field.source_id()]; + std::unique_ptr<PartitionColumnTransform> partition_column_transform = + PartitionColumnTransforms::create( + partition_field, _vec_output_expr_ctxs[column_idx]->root()->type()); + partition_columns.emplace_back(partition_field, + _vec_output_expr_ctxs[column_idx]->root()->type(), + column_idx, std::move(partition_column_transform)); + } + return partition_columns; +} + +Status VIcebergTableWriter::write(vectorized::Block& block) { Review Comment: warning: function 'write' has cognitive complexity of 81 (threshold 50) [readability-function-cognitive-complexity] ```cpp Status VIcebergTableWriter::write(vectorized::Block& block) { ^ ``` <details> <summary>Additional context</summary> **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:112:** +1, including nesting penalty of 0, nesting level increased to 1 ```cpp if (_iceberg_partition_columns.empty()) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:117:** +2, including nesting penalty of 1, nesting level increased to 2 ```cpp if (writer_iter == _partitions_to_writers.end()) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:120:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp } catch (doris::Exception& e) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:124:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp RETURN_IF_ERROR(writer->open(_state, _profile)); ^ ``` **be/src/common/status.h:616:** expanded from macro 'RETURN_IF_ERROR' ```cpp do { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:124:** +4, including nesting penalty of 3, nesting level increased to 4 ```cpp RETURN_IF_ERROR(writer->open(_state, _profile)); ^ ``` **be/src/common/status.h:618:** expanded from macro 'RETURN_IF_ERROR' ```cpp if (UNLIKELY(!_status_.ok())) { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:125:** +1, nesting level increased to 2 ```cpp } else { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:126:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp if (writer_iter->second->written_len() > config::iceberg_sink_max_file_size) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:137:** +4, including nesting penalty of 3, nesting level increased to 4 ```cpp } catch (doris::Exception& e) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:141:** +4, including nesting penalty of 3, nesting level increased to 4 ```cpp RETURN_IF_ERROR(writer->open(_state, _profile)); ^ ``` **be/src/common/status.h:616:** expanded from macro 'RETURN_IF_ERROR' ```cpp do { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:141:** +5, including nesting penalty of 4, nesting level increased to 5 ```cpp RETURN_IF_ERROR(writer->open(_state, _profile)); ^ ``` **be/src/common/status.h:618:** expanded from macro 'RETURN_IF_ERROR' ```cpp if (UNLIKELY(!_status_.ok())) { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:142:** +1, nesting level increased to 3 ```cpp } else { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:148:** +2, including nesting penalty of 1, nesting level increased to 2 ```cpp RETURN_IF_ERROR(writer->write(block)); ^ ``` **be/src/common/status.h:616:** expanded from macro 'RETURN_IF_ERROR' ```cpp do { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:148:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp RETURN_IF_ERROR(writer->write(block)); ^ ``` **be/src/common/status.h:618:** expanded from macro 'RETURN_IF_ERROR' ```cpp if (UNLIKELY(!_status_.ok())) { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:168:** +1, including nesting penalty of 0, nesting level increased to 1 ```cpp for (int i = 0; i < block.rows(); ++i) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:172:** +2, including nesting penalty of 1, nesting level increased to 2 ```cpp } catch (doris::Exception& e) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:179:** +2, including nesting penalty of 1, nesting level increased to 2 ```cpp } catch (doris::Exception& e) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:183:** nesting level increased to 2 ```cpp [&](const std::string& partition_name, int position, ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:189:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp RETURN_IF_ERROR(writer->open(_state, _profile)); ^ ``` **be/src/common/status.h:616:** expanded from macro 'RETURN_IF_ERROR' ```cpp do { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:189:** +4, including nesting penalty of 3, nesting level increased to 4 ```cpp RETURN_IF_ERROR(writer->open(_state, _profile)); ^ ``` **be/src/common/status.h:618:** expanded from macro 'RETURN_IF_ERROR' ```cpp if (UNLIKELY(!_status_.ok())) { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:195:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp } catch (doris::Exception& e) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:202:** +2, including nesting penalty of 1, nesting level increased to 2 ```cpp if (writer_iter == _partitions_to_writers.end()) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:204:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp if (_partitions_to_writers.size() + 1 > ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:210:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); ^ ``` **be/src/common/status.h:616:** expanded from macro 'RETURN_IF_ERROR' ```cpp do { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:210:** +4, including nesting penalty of 3, nesting level increased to 4 ```cpp RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); ^ ``` **be/src/common/status.h:618:** expanded from macro 'RETURN_IF_ERROR' ```cpp if (UNLIKELY(!_status_.ok())) { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:211:** +1, nesting level increased to 2 ```cpp } else { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:213:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp if (writer_iter->second->written_len() > config::iceberg_sink_max_file_size) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:222:** +4, including nesting penalty of 3, nesting level increased to 4 ```cpp RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, ^ ``` **be/src/common/status.h:616:** expanded from macro 'RETURN_IF_ERROR' ```cpp do { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:222:** +5, including nesting penalty of 4, nesting level increased to 5 ```cpp RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, ^ ``` **be/src/common/status.h:618:** expanded from macro 'RETURN_IF_ERROR' ```cpp if (UNLIKELY(!_status_.ok())) { \ ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:224:** +1, nesting level increased to 3 ```cpp } else { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:228:** +3, including nesting penalty of 2, nesting level increased to 3 ```cpp if (writer_pos_iter == writer_positions.end()) { ^ ``` **be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:232:** +1, nesting level increased to 3 ```cpp } else { ^ ``` </details> ########## be/test/vec/exec/format/table/iceberg/schema_parser_test.cpp: ########## @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exec/format/table/iceberg/schema_parser.h" + +#include <gtest/gtest.h> + +#include "vec/exec/format/table/iceberg/schema.h" + +namespace doris { +namespace iceberg { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::iceberg { ``` be/test/vec/exec/format/table/iceberg/schema_parser_test.cpp:262: ```diff - } // namespace iceberg - } // namespace doris + } // namespace doris ``` ########## be/test/vec/exec/format/table/iceberg/type_test.cpp: ########## @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gtest/gtest.h> Review Comment: warning: 'gtest/gtest.h' file not found [clang-diagnostic-error] ```cpp #include <gtest/gtest.h> ^ ``` ########## be/test/vec/exec/format/table/iceberg/partition_spec_parser_test.cpp: ########## @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exec/format/table/iceberg/partition_spec_parser.h" + +#include <gtest/gtest.h> + +#include "vec/exec/format/table/iceberg/schema.h" +#include "vec/exec/format/table/iceberg/types.h" + +namespace doris { +namespace iceberg { Review Comment: warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces] ```suggestion namespace doris::iceberg { ``` be/test/vec/exec/format/table/iceberg/partition_spec_parser_test.cpp:85: ```diff - } // namespace iceberg - } // namespace doris + } // namespace doris ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
