This is an automated email from the ASF dual-hosted git repository.
stalary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4023670f35 [BugFix](DOE) Add http prefix when it's not set in hosts
properties. (#17745)
4023670f35 is described below
commit 4023670f351b1e800ec996c5e237d63aa260d7ab
Author: huangzhaowei <huangzhaowei....@bytedance.com>
AuthorDate: Tue Mar 21 10:08:20 2023 +0800
[BugFix](DOE) Add http prefix when it's not set in hosts properties.
(#17745)
* Add http prefix when it's not set in hosts properties
---
.../java/org/apache/doris/catalog/EsResource.java | 9 ++
.../java/org/apache/doris/catalog/EsTable.java | 8 +-
.../apache/doris/datasource/EsExternalCatalog.java | 6 +-
.../data/es_p0/test_es_query_no_http_url.out | 16 +++
.../suites/es_p0/test_es_query_no_http_url.groovy | 160 +++++++++++++++++++++
5 files changed, 196 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
index 91cb4529ad..84f51cb820 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
@@ -136,6 +136,15 @@ public class EsResource extends Resource {
}
}
+ public static void fillUrlsWithSchema(String[] urls, boolean isSslEnabled)
{
+ for (int i = 0; i < urls.length; i++) {
+ String seed = urls[i].trim();
+ if (!seed.startsWith("http://";) && !seed.startsWith("https://";)) {
+ urls[i] = (isSslEnabled ? "https://"; : "http://";) + seed;
+ }
+ }
+ }
+
private Map<String, String> processCompatibleProperties(Map<String,
String> props) {
// Compatible with ES catalog properties
Map<String, String> properties = Maps.newHashMap(props);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index 5d77163857..b9dbc6ff64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -151,8 +151,6 @@ public class EsTable extends Table {
private void validate(Map<String, String> properties) throws DdlException {
EsResource.valid(properties, false);
- hosts = properties.get(EsResource.HOSTS).trim();
- seeds = hosts.split(",");
if (properties.containsKey(EsResource.USER)) {
userName = properties.get(EsResource.USER).trim();
}
@@ -198,6 +196,12 @@ public class EsTable extends Table {
maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
}
}
+
+ hosts = properties.get(EsResource.HOSTS).trim();
+ seeds = hosts.split(",");
+ // parse httpSslEnabled before use it here.
+ EsResource.fillUrlsWithSchema(seeds, httpSslEnabled);
+
tableContext.put("hosts", hosts);
tableContext.put("userName", userName);
tableContext.put("passwd", passwd);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
index 650e5374e5..52f9bc984f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
@@ -70,7 +70,11 @@ public class EsExternalCatalog extends ExternalCatalog {
public String[] getNodes() {
String hosts = catalogProperty.getOrDefault(EsResource.HOSTS, "");
- return hosts.trim().split(",");
+ String sslEnabled =
+ catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED,
EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE);
+ String[] hostUrls = hosts.trim().split(",");
+ EsResource.fillUrlsWithSchema(hostUrls,
Boolean.parseBoolean(sslEnabled));
+ return hostUrls;
}
public String getUsername() {
diff --git a/regression-test/data/es_p0/test_es_query_no_http_url.out
b/regression-test/data/es_p0/test_es_query_no_http_url.out
new file mode 100644
index 0000000000..b195d6d187
--- /dev/null
+++ b/regression-test/data/es_p0/test_es_query_no_http_url.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql51 --
+[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f']
[128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4]
2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02]
3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c']
['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10
2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1]
[32768, 32769, -32769, -32770]
+
+-- !sql52 --
+[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f']
[128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4]
2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02]
3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c']
['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10
2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1]
[32768, 32769, -32769, -32770]
+
+-- !sql61 --
+[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00]
['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2,
1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1',
'127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}',
'{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e',
'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00
+
+-- !sql71 --
+[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1,
2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1']
['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}',
'{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e',
'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00
2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10
2022-08-08T20:10:10
+
+-- !sql81 --
+[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1,
2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1']
['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}',
'{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e',
'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00
2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10
2022-08-08T20:10:10
+
diff --git a/regression-test/suites/es_p0/test_es_query_no_http_url.groovy
b/regression-test/suites/es_p0/test_es_query_no_http_url.groovy
new file mode 100644
index 0000000000..2651626672
--- /dev/null
+++ b/regression-test/suites/es_p0/test_es_query_no_http_url.groovy
@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_es_query_no_http_url", "p0") {
+
+ String enabled = context.config.otherConfigs.get("enableEsTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String es_6_port = context.config.otherConfigs.get("es_6_port")
+ String es_7_port = context.config.otherConfigs.get("es_7_port")
+ String es_8_port = context.config.otherConfigs.get("es_8_port")
+
+ sql """drop catalog if exists es6;"""
+ sql """drop catalog if exists es7;"""
+ sql """drop catalog if exists es8;"""
+ sql """drop resource if exists es6_resource;"""
+ sql """drop resource if exists es7_resource;"""
+ sql """drop resource if exists es8_resource;"""
+ sql """drop table if exists test_v1;"""
+ sql """drop table if exists test_v2;"""
+
+ // test old create-catalog syntax for compatibility
+ sql """
+ create catalog es6
+ properties (
+ "type"="es",
+ "elasticsearch.hosts"="127.0.0.1:$es_6_port",
+ "elasticsearch.nodes_discovery"="false",
+ "elasticsearch.keyword_sniff"="true"
+ );
+ """
+
+ // test new create catalog syntax
+ sql """create resource if not exists es7_resource properties(
+ "type"="es",
+ "hosts"="127.0.0.1:$es_7_port",
+ "nodes_discovery"="false",
+ "enable_keyword_sniff"="true"
+ );
+ """
+
+ sql """create resource if not exists es8_resource properties(
+ "type"="es",
+ "hosts"="127.0.0.1:$es_8_port",
+ "nodes_discovery"="false",
+ "enable_keyword_sniff"="true"
+ );
+ """
+
+ // test external table for datetime
+ sql """
+ CREATE TABLE `test_v1` (
+ `c_datetime` array<datev2> NULL,
+ `c_long` array<bigint(20)> NULL,
+ `c_unsigned_long` array<largeint(40)> NULL,
+ `c_text` array<text> NULL,
+ `c_short` array<smallint(6)> NULL,
+ `c_ip` array<text> NULL,
+ `test1` text NULL,
+ `c_half_float` array<float> NULL,
+ `test4` date NULL,
+ `test5` datetime NULL,
+ `test2` text NULL,
+ `c_date` array<datev2> NULL,
+ `test3` double NULL,
+ `c_scaled_float` array<double> NULL,
+ `c_float` array<float> NULL,
+ `c_double` array<double> NULL,
+ `c_keyword` array<text> NULL,
+ `c_person` array<text> NULL,
+ `test6` datetime NULL,
+ `test7` datetime NULL,
+ `test8` datetime NULL,
+ `c_byte` array<tinyint(4)> NULL,
+ `c_bool` array<boolean> NULL,
+ `c_integer` array<int(11)> NULL
+ ) ENGINE=ELASTICSEARCH
+ COMMENT 'ELASTICSEARCH'
+ PROPERTIES (
+ "hosts" = "127.0.0.1:$es_7_port",
+ "index" = "test1",
+ "nodes_discovery"="false",
+ "enable_keyword_sniff"="true",
+ "http_ssl_enabled"="false"
+ );
+ """
+ order_qt_sql51 """select * from test_v1 where test2='text#1'"""
+
+ sql """
+ CREATE TABLE `test_v2` (
+ `c_datetime` array<datev2> NULL,
+ `c_long` array<bigint(20)> NULL,
+ `c_unsigned_long` array<largeint(40)> NULL,
+ `c_text` array<text> NULL,
+ `c_short` array<smallint(6)> NULL,
+ `c_ip` array<text> NULL,
+ `test1` text NULL,
+ `c_half_float` array<float> NULL,
+ `test4` datev2 NULL,
+ `test5` datetimev2 NULL,
+ `test2` text NULL,
+ `c_date` array<datev2> NULL,
+ `test3` double NULL,
+ `c_scaled_float` array<double> NULL,
+ `c_float` array<float> NULL,
+ `c_double` array<double> NULL,
+ `c_keyword` array<text> NULL,
+ `c_person` array<text> NULL,
+ `test6` datetimev2 NULL,
+ `test7` datetimev2 NULL,
+ `test8` datetimev2 NULL,
+ `c_byte` array<tinyint(4)> NULL,
+ `c_bool` array<boolean> NULL,
+ `c_integer` array<int(11)> NULL
+ ) ENGINE=ELASTICSEARCH
+ COMMENT 'ELASTICSEARCH'
+ PROPERTIES (
+ "hosts" = "127.0.0.1:$es_8_port",
+ "index" = "test1",
+ "nodes_discovery"="false",
+ "enable_keyword_sniff"="true",
+ "http_ssl_enabled"="false"
+ );
+ """
+ order_qt_sql52 """select * from test_v2 where test2='text#1'"""
+
+ sql """create catalog if not exists es6 with resource es6_resource;"""
+ sql """create catalog if not exists es7 with resource es7_resource;"""
+ sql """create catalog if not exists es8 with resource es8_resource;"""
+ // es6
+ sql """switch es6"""
+ order_qt_sql61 """select * from test1 where test2='text#1'"""
+ // es7
+ sql """switch es7"""
+ order_qt_sql71 """select * from test1 where test2='text#1'"""
+ // es8
+ sql """switch es8"""
+ order_qt_sql81 """select * from test1 where test2='text#1'"""
+
+ sql """drop catalog if exists es6;"""
+ sql """drop catalog if exists es7;"""
+ sql """drop catalog if exists es8;"""
+ sql """drop resource if exists es6_resource;"""
+ sql """drop resource if exists es7_resource;"""
+ sql """drop resource if exists es8_resource;"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
