This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e841bb1138584f463cc14935eb97348d60896649
Author: zhangdong <493738...@qq.com>
AuthorDate: Mon Sep 11 18:32:31 2023 +0800
[Enhance](ip)optimize priority_ network matching logic for be (#23795)
Issue Number: close #xxx
If the user has configured the wrong priority_network, direct startup
failure to avoid users mistakenly assuming that the configuration is correct
If the user has not configured p_ n. Select only the first IP from the IPv4
list, rather than selecting from all IPs, to avoid users' servers not
supporting IPv4
extends #23784
---
be/src/service/backend_options.cpp | 83 ++++++++++++++++++-------------
be/src/service/backend_options.h | 6 ++-
be/src/vec/exec/scan/new_es_scan_node.cpp | 4 +-
be/test/util/backend_options_test.cpp | 78 +++++++++++++++++++++++++++++
4 files changed, 134 insertions(+), 37 deletions(-)
diff --git a/be/src/service/backend_options.cpp
b/be/src/service/backend_options.cpp
index 20f4e68c76..c832573336 100644
--- a/be/src/service/backend_options.cpp
+++ b/be/src/service/backend_options.cpp
@@ -38,7 +38,7 @@ bool BackendOptions::_bind_ipv6 = false;
const char* _service_bind_address = "0.0.0.0";
bool BackendOptions::init() {
- if (!analyze_priority_cidrs()) {
+ if (!analyze_priority_cidrs(config::priority_networks,
&_s_priority_cidrs)) {
return false;
}
std::vector<InetAddress> hosts;
@@ -53,37 +53,12 @@ bool BackendOptions::init() {
LOG(FATAL) << "failed to get host";
return false;
}
-
- std::string loopback;
- std::vector<InetAddress>::iterator addr_it = hosts.begin();
- for (; addr_it != hosts.end(); ++addr_it) {
- VLOG_CRITICAL << "check ip=" << addr_it->get_host_address();
- if (!_s_priority_cidrs.empty()) {
- // Whether to use IPV4 or IPV6, it's configured by CIDR format.
- // If both IPV4 and IPV6 are configured, the config order decides
priority.
- if (is_in_prior_network(addr_it->get_host_address())) {
- _s_localhost = addr_it->get_host_address();
- _bind_ipv6 = addr_it->is_ipv6();
- break;
- }
- LOG(INFO) << "skip ip not belonged to priority networks: "
- << addr_it->get_host_address();
- } else if ((*addr_it).is_loopback()) {
- loopback = addr_it->get_host_address();
- _bind_ipv6 = addr_it->is_ipv6();
- } else {
- _s_localhost = addr_it->get_host_address();
- _bind_ipv6 = addr_it->is_ipv6();
- break;
- }
+ if (!analyze_localhost(_s_localhost, _bind_ipv6, &_s_priority_cidrs,
&hosts)) {
+ return false;
}
if (_bind_ipv6) {
_service_bind_address = "[::0]";
}
- if (_s_localhost.empty()) {
- LOG(INFO) << "fail to find one valid non-loopback address, use
loopback address.";
- _s_localhost = loopback;
- }
LOG(INFO) << "local host ip=" << _s_localhost;
return true;
}
@@ -118,14 +93,14 @@ const char*
BackendOptions::get_service_bind_address_without_bracket() {
return _service_bind_address;
}
-bool BackendOptions::analyze_priority_cidrs() {
- if (config::priority_networks == "") {
+bool BackendOptions::analyze_priority_cidrs(const std::string&
priority_networks,
+ std::vector<CIDR>* cidrs) {
+ if (priority_networks == "") {
return true;
}
- LOG(INFO) << "priority cidrs in conf: " << config::priority_networks;
+ LOG(INFO) << "priority cidrs: " << priority_networks;
- std::vector<std::string> cidr_strs =
- strings::Split(config::priority_networks, PRIORITY_CIDR_SEPARATOR);
+ std::vector<std::string> cidr_strs = strings::Split(priority_networks,
PRIORITY_CIDR_SEPARATOR);
for (auto& cidr_str : cidr_strs) {
CIDR cidr;
@@ -133,7 +108,47 @@ bool BackendOptions::analyze_priority_cidrs() {
LOG(FATAL) << "wrong cidr format. cidr_str=" << cidr_str;
return false;
}
- _s_priority_cidrs.push_back(cidr);
+ cidrs->push_back(cidr);
+ }
+ return true;
+}
+
+bool BackendOptions::analyze_localhost(std::string& localhost, bool& bind_ipv6,
+ std::vector<CIDR>* cidrs,
std::vector<InetAddress>* hosts) {
+ std::vector<InetAddress>::iterator addr_it = hosts->begin();
+ if (!cidrs->empty()) {
+ for (; addr_it != hosts->end(); ++addr_it) {
+ VLOG_CRITICAL << "check ip=" << addr_it->get_host_address();
+ // Whether to use IPV4 or IPV6, it's configured by CIDR format.
+ // If both IPV4 and IPV6 are configured, the config order decides
priority.
+ if (is_in_prior_network(addr_it->get_host_address())) {
+ localhost = addr_it->get_host_address();
+ bind_ipv6 = addr_it->is_ipv6();
+ break;
+ }
+ LOG(INFO) << "skip ip not belonged to priority networks: "
+ << addr_it->get_host_address();
+ }
+ if (localhost.empty()) {
+ LOG(FATAL) << "fail to find one valid address, exit.";
+ return false;
+ }
+ } else {
+ std::string loopback;
+ for (; addr_it != hosts->end(); ++addr_it) {
+ if ((*addr_it).is_loopback()) {
+ loopback = addr_it->get_host_address();
+ _bind_ipv6 = addr_it->is_ipv6();
+ } else if (!addr_it->is_ipv6()) {
+ localhost = addr_it->get_host_address();
+ _bind_ipv6 = addr_it->is_ipv6();
+ break;
+ }
+ }
+ if (localhost.empty()) {
+ LOG(INFO) << "fail to find one valid non-loopback address, use
loopback address.";
+ localhost = loopback;
+ }
}
return true;
}
diff --git a/be/src/service/backend_options.h b/be/src/service/backend_options.h
index 3aff934036..7229337388 100644
--- a/be/src/service/backend_options.h
+++ b/be/src/service/backend_options.h
@@ -23,6 +23,7 @@
#include <vector>
#include "gen_cpp/Types_types.h"
+#include "util/network_util.h"
namespace doris {
@@ -37,9 +38,12 @@ public:
static bool is_bind_ipv6();
static const char* get_service_bind_address();
static const char* get_service_bind_address_without_bracket();
+ static bool analyze_priority_cidrs(const std::string& priority_networks,
+ std::vector<CIDR>* cidrs);
+ static bool analyze_localhost(std::string& localhost, bool& bind_ipv6,
std::vector<CIDR>* cidrs,
+ std::vector<InetAddress>* hosts);
private:
- static bool analyze_priority_cidrs();
static bool is_in_prior_network(const std::string& ip);
static std::string _s_localhost;
diff --git a/be/src/vec/exec/scan/new_es_scan_node.cpp
b/be/src/vec/exec/scan/new_es_scan_node.cpp
index 088784b330..6a3ec3a738 100644
--- a/be/src/vec/exec/scan/new_es_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_es_scan_node.cpp
@@ -41,7 +41,7 @@ class VScanner;
static const std::string NEW_SCAN_NODE_TYPE = "NewEsScanNode";
// Prefer to the local host
-static std::string get_host_port(const std::vector<doris::TNetworkAddress>&
es_hosts) {
+static std::string get_host_and_port(const
std::vector<doris::TNetworkAddress>& es_hosts) {
std::string host_port;
std::string localhost = doris::BackendOptions::get_localhost();
@@ -152,7 +152,7 @@ Status
NewEsScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
}
properties[ESScanReader::KEY_SHARD] =
std::to_string(es_scan_range->shard_id);
properties[ESScanReader::KEY_BATCH_SIZE] =
std::to_string(_state->batch_size());
- properties[ESScanReader::KEY_HOST_PORT] =
get_host_port(es_scan_range->es_hosts);
+ properties[ESScanReader::KEY_HOST_PORT] =
get_host_and_port(es_scan_range->es_hosts);
// push down limit to Elasticsearch
// if predicate in _conjunct_ctxs can not be processed by
Elasticsearch, we can not push down limit operator to Elasticsearch
if (limit() != -1 && limit() <= _state->batch_size()) {
diff --git a/be/test/util/backend_options_test.cpp
b/be/test/util/backend_options_test.cpp
new file mode 100644
index 0000000000..a8d0ca4151
--- /dev/null
+++ b/be/test/util/backend_options_test.cpp
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "service/backend_options.h"
+
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+
+#include "gtest/gtest_pred_impl.h"
+#include "util/cidr.h"
+
+namespace doris {
+
+class BackendOptionsTest : public testing::Test {
+public:
+ BackendOptionsTest() {}
+ virtual ~BackendOptionsTest() {}
+};
+
+// only loopback
+TEST_F(BackendOptionsTest, emptyCidr1) {
+ std::vector<InetAddress> hosts;
+ hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true);
+
+ std::vector<CIDR> cidrs;
+ BackendOptions::analyze_priority_cidrs("", &cidrs);
+ std::string localhost;
+ bool bind_ipv6 = false;
+ BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts);
+ EXPECT_STREQ("127.0.0.1", localhost.c_str());
+}
+
+// priority not loopback
+TEST_F(BackendOptionsTest, emptyCidr2) {
+ std::vector<InetAddress> hosts;
+ hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true);
+ hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false);
+ hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false);
+
+ std::vector<CIDR> cidrs;
+ BackendOptions::analyze_priority_cidrs("", &cidrs);
+ std::string localhost;
+ bool bind_ipv6 = false;
+ BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts);
+ EXPECT_STREQ("10.10.10.10", localhost.c_str());
+}
+
+// not choose ipv6
+TEST_F(BackendOptionsTest, emptyCidr3) {
+ std::vector<InetAddress> hosts;
+ hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true);
+ hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6,
false);
+ hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false);
+ hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false);
+
+ std::vector<CIDR> cidrs;
+ BackendOptions::analyze_priority_cidrs("", &cidrs);
+ std::string localhost;
+ bool bind_ipv6 = false;
+ BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts);
+ EXPECT_STREQ("10.10.10.10", localhost.c_str());
+}
+
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
