This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d06ff2c827c8810978854008227eecb4789093fe
Author: qiye <jianliang5...@gmail.com>
AuthorDate: Mon Dec 5 08:33:51 2022 +0800
[fix](DOE) Support ES index which contains dynamic_templates (#14762)
Support ES index with dynamic_templates. And do not support index mapping
without explicit mapping.
---
.../doris/external/elasticsearch/EsUtil.java | 23 ++++++++-
.../doris/external/elasticsearch/EsUtilTest.java | 52 +++++++++++++++++++
.../data/es/es6_dynamic_templates_mapping.json | 60 ++++++++++++++++++++++
.../es/es6_only_dynamic_templates_mapping.json | 40 +++++++++++++++
.../data/es/es7_dynamic_templates_mapping.json | 58 +++++++++++++++++++++
.../es/es7_only_dynamic_templates_mapping.json | 38 ++++++++++++++
.../data/es/es8_dynamic_templates_mapping.json | 58 +++++++++++++++++++++
.../es/es8_only_dynamic_templates_mapping.json | 38 ++++++++++++++
8 files changed, 365 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 1d50bbca42..7cf22cfda3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -163,22 +163,41 @@ public class EsUtil {
// 2. Multi-catalog auto infer
// 3. Equal 6.8.x and before user not passed
if (mappingType == null) {
+ // remove dynamic templates, for ES 7.x and 8.x
+ checkDynamicTemplates(mappings);
String firstType = (String) mappings.keySet().iterator().next();
if (!"properties".equals(firstType)) {
// If type is not passed in takes the first type.
- return (JSONObject) mappings.get(firstType);
+ JSONObject firstData = (JSONObject) mappings.get(firstType);
+ // check for ES 6.x and before
+ checkDynamicTemplates(firstData);
+ return firstData;
}
// Equal 7.x and after
return mappings;
} else {
if (mappings.containsKey(mappingType)) {
- return (JSONObject) mappings.get(mappingType);
+ JSONObject jsonData = (JSONObject) mappings.get(mappingType);
+ // check for ES 6.x and before
+ checkDynamicTemplates(jsonData);
+ return jsonData;
}
// Compatible type error
return getRootSchema(mappings, null);
}
}
+ /**
+ * Remove `dynamic_templates` and check explicit mapping
+ * @param mappings
+ */
+ private static void checkDynamicTemplates(JSONObject mappings) {
+ mappings.remove("dynamic_templates");
+ if (mappings.isEmpty()) {
+ throw new DorisEsException("Do not support index without explicit
mapping.");
+ }
+ }
+
/**
* Get mapping properties JSONObject.
**/
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 654cd08697..7783b4d437 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -41,8 +41,10 @@ import mockit.Injectable;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
+import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -68,6 +70,9 @@ public class EsUtilTest extends EsTestCase {
+ " \"created\": \"5050099\"\n" + "
}\n" + " }\n"
+ " }}";
+ @Rule
+ public ExpectedException expectedEx = ExpectedException.none();
+
/**
* Init columns.
**/
@@ -298,17 +303,36 @@ public class EsUtilTest extends EsTestCase {
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliases.toJSONString());
+
JSONObject testAliasesNoType = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_aliases_mapping.json"), null);
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliasesNoType.toJSONString());
+
JSONObject testIndex = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_index_mapping.json"),
"doc");
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
+
+ JSONObject testDynamicTemplates = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_dynamic_templates_mapping.json"),
+ "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
+ testDynamicTemplates.toJSONString());
+
+ expectedEx.expect(DorisEsException.class);
+ expectedEx.expectMessage("Do not support index without explicit
mapping.");
+ EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_only_dynamic_templates_mapping.json"),
+ "doc");
+
+ expectedEx.expect(DorisEsException.class);
+ expectedEx.expectMessage("Do not support index without explicit
mapping.");
+ EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_only_dynamic_templates_mapping.json"),
+ null);
}
@Test
@@ -318,17 +342,31 @@ public class EsUtilTest extends EsTestCase {
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliases.toJSONString());
+
JSONObject testAliasesErrorType = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es7_aliases_mapping.json"), "doc");
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliasesErrorType.toJSONString());
+
JSONObject testIndex = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es7_index_mapping.json"),
"doc");
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
+
+ JSONObject testDynamicTemplates = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es7_dynamic_templates_mapping.json"),
+ null);
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
+ testDynamicTemplates.toJSONString());
+
+ expectedEx.expect(DorisEsException.class);
+ expectedEx.expectMessage("Do not support index without explicit
mapping.");
+ EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es7_only_dynamic_templates_mapping.json"),
+ null);
}
@Test
@@ -338,17 +376,31 @@ public class EsUtilTest extends EsTestCase {
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliases.toJSONString());
+
JSONObject testAliasesErrorType = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es8_aliases_mapping.json"), "doc");
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliasesErrorType.toJSONString());
+
JSONObject testIndex = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es8_index_mapping.json"),
"doc");
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
+
+ JSONObject testDynamicTemplates = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es8_dynamic_templates_mapping.json"),
+ "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
+ testDynamicTemplates.toJSONString());
+
+ expectedEx.expect(DorisEsException.class);
+ expectedEx.expectMessage("Do not support index without explicit
mapping.");
+ EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es8_only_dynamic_templates_mapping.json"),
+ "doc");
}
}
diff --git
a/fe/fe-core/src/test/resources/data/es/es6_dynamic_templates_mapping.json
b/fe/fe-core/src/test/resources/data/es/es6_dynamic_templates_mapping.json
new file mode 100644
index 0000000000..ccde761b6b
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es6_dynamic_templates_mapping.json
@@ -0,0 +1,60 @@
+{
+ "test_202207": {
+ "mappings": {
+ "doc": {
+ "dynamic_templates" : [
+ {
+ "message_full" : {
+ "match" : "message_full",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 2048,
+ "type" : "keyword"
+ }
+ },
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "message" : {
+ "match" : "message",
+ "mapping" : {
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "strings" : {
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/resources/data/es/es6_only_dynamic_templates_mapping.json
b/fe/fe-core/src/test/resources/data/es/es6_only_dynamic_templates_mapping.json
new file mode 100644
index 0000000000..6bc825320c
--- /dev/null
+++
b/fe/fe-core/src/test/resources/data/es/es6_only_dynamic_templates_mapping.json
@@ -0,0 +1,40 @@
+{
+ "test_202207": {
+ "mappings": {
+ "doc": {
+ "dynamic_templates" : [
+ {
+ "message_full" : {
+ "match" : "message_full",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 2048,
+ "type" : "keyword"
+ }
+ },
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "message" : {
+ "match" : "message",
+ "mapping" : {
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "strings" : {
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/resources/data/es/es7_dynamic_templates_mapping.json
b/fe/fe-core/src/test/resources/data/es/es7_dynamic_templates_mapping.json
new file mode 100644
index 0000000000..5a5598a688
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es7_dynamic_templates_mapping.json
@@ -0,0 +1,58 @@
+{
+ "test_202207": {
+ "mappings": {
+ "dynamic_templates" : [
+ {
+ "message_full" : {
+ "match" : "message_full",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 2048,
+ "type" : "keyword"
+ }
+ },
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "message" : {
+ "match" : "message",
+ "mapping" : {
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "strings" : {
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/resources/data/es/es7_only_dynamic_templates_mapping.json
b/fe/fe-core/src/test/resources/data/es/es7_only_dynamic_templates_mapping.json
new file mode 100644
index 0000000000..091c4b0def
--- /dev/null
+++
b/fe/fe-core/src/test/resources/data/es/es7_only_dynamic_templates_mapping.json
@@ -0,0 +1,38 @@
+{
+ "test_202207": {
+ "mappings": {
+ "dynamic_templates" : [
+ {
+ "message_full" : {
+ "match" : "message_full",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 2048,
+ "type" : "keyword"
+ }
+ },
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "message" : {
+ "match" : "message",
+ "mapping" : {
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "strings" : {
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ ]
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/resources/data/es/es8_dynamic_templates_mapping.json
b/fe/fe-core/src/test/resources/data/es/es8_dynamic_templates_mapping.json
new file mode 100644
index 0000000000..5a5598a688
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es8_dynamic_templates_mapping.json
@@ -0,0 +1,58 @@
+{
+ "test_202207": {
+ "mappings": {
+ "dynamic_templates" : [
+ {
+ "message_full" : {
+ "match" : "message_full",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 2048,
+ "type" : "keyword"
+ }
+ },
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "message" : {
+ "match" : "message",
+ "mapping" : {
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "strings" : {
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/resources/data/es/es8_only_dynamic_templates_mapping.json
b/fe/fe-core/src/test/resources/data/es/es8_only_dynamic_templates_mapping.json
new file mode 100644
index 0000000000..091c4b0def
--- /dev/null
+++
b/fe/fe-core/src/test/resources/data/es/es8_only_dynamic_templates_mapping.json
@@ -0,0 +1,38 @@
+{
+ "test_202207": {
+ "mappings": {
+ "dynamic_templates" : [
+ {
+ "message_full" : {
+ "match" : "message_full",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 2048,
+ "type" : "keyword"
+ }
+ },
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "message" : {
+ "match" : "message",
+ "mapping" : {
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "strings" : {
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ ]
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
