dataroaring commented on code in PR #18874: URL: https://github.com/apache/doris/pull/18874#discussion_r1187419385
########## be/src/util/open_partition_closure.h: ########## @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <google/protobuf/stubs/common.h> + +#include <atomic> + +#include "runtime/exec_env.h" +#include "runtime/thread_context.h" +#include "service/brpc.h" +#include "vec/sink/vtablet_sink.h" + +namespace doris { + +using namespace stream_load; + +template <typename T> +class OpenPartitionClosure : public google::protobuf::Closure { +public: + OpenPartitionClosure(VNodeChannel* vnode_channel, IndexChannel* index_channel) + : vnode_channel(vnode_channel), index_channel(index_channel) {} + ~OpenPartitionClosure() = default; + + void Run() override { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->orphan_mem_tracker()); + if (cntl.Failed()) { + std::stringstream ss; + ss << "failed to open partition, error=" << berror(this->cntl.ErrorCode()) + << ", error_text=" << this->cntl.ErrorText(); + LOG(WARNING) << ss.str() << " " << vnode_channel->channel_info(); + vnode_channel->cancel("Open partition error"); + index_channel->mark_as_failed(vnode_channel->node_id(), vnode_channel->host(), Review Comment: we should retry until load is timeout. ########## docs/en/docs/admin-manual/config/be-config.md: ########## @@ -1227,12 +1227,30 @@ Metrics: {"filtered_rows":0,"input_row_num":3346807,"input_rowsets_count":42,"in - The RPC timeout for sending a Batch (1024 lines) during import. The default is 60 seconds. Since this RPC may involve writing multiple batches of memory, the RPC timeout may be caused by writing batches, so this timeout can be adjusted to reduce timeout errors (such as send batch fail errors). Also, if you increase the write_buffer_size configuration, you need to increase this parameter as well. * Default value: 60 +#### `open_partition_rpc_timeout_sec` + +* Description: Update interval of partition state cache + - The RPC timeout for sending a Batch (1024 lines) during import. The default is 60 seconds. Since this RPC may involve writing multiple batches of memory, the RPC timeout may be caused by writing batches, so this timeout can be adjusted to reduce timeout errors (such as send batch fail errors). Also, if you increase the write_buffer_size configuration, you need to increase this parameter as well. +* Default value: 60 + #### `tablet_writer_ignore_eovercrowded` * Type: bool * Description: Used to ignore brpc error '[E1011]The server is overcrowded' when writing data. * Default value: false +#### `enable_lazy_open_partition` + +* Type: bool +* Description: When importing, most partitions may not need to be written, and lazy opening can be used to only open the partitions that need to be written.When there is mixed deployment in the upgraded version, it needs to be set to false. +* Default value: true + +#### `open_partition_ignore_eovercrowded` + +* Type: bool +* Description: Used to ignore brpc error '[E1011]The server is overcrowded' when writing data. +* Default value: false Review Comment: set ignore by default and remove the config. ########## be/src/runtime/tablets_channel.cpp: ########## @@ -107,12 +107,22 @@ Status TabletsChannel::open(const PTabletWriterOpenRequest& request) { _next_seqs.resize(_num_remaining_senders, 0); _closed_senders.Reset(_num_remaining_senders); - RETURN_IF_ERROR(_open_all_writers(request)); - + if (!config::enable_lazy_open_partition) { + RETURN_IF_ERROR(_open_all_writers(request)); Review Comment: If we are rolling upgrade doris, then may be sink's works as old version and tablets_channel works as latest version, then load breaks. ########## be/src/vec/sink/vtablet_sink.h: ########## @@ -599,6 +607,9 @@ class VOlapTableSink final : public DataSink { std::vector<vectorized::VExprContext*> _output_vexpr_ctxs; RuntimeState* _state = nullptr; + + std::unordered_map<int64_t,bool> opened_partitions; Review Comment: _opened_partitions -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
