This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5d9dfe67c8b97be8e4775b47536d8ccf73a44744
Author: Stalary <stal...@163.com>
AuthorDate: Wed Jan 11 17:00:59 2023 +0800
[fix](DOE): Fix query _id error and es properties error (#15792)
Fix query _id error
_id not exist mapping, but be can query it, we need skip check it exist
mapping.
---
.../java/org/apache/doris/catalog/EsResource.java | 4 +++
.../java/org/apache/doris/catalog/EsTable.java | 36 +++++++++-------------
.../apache/doris/datasource/EsExternalCatalog.java | 28 ++++++-----------
.../doris/external/elasticsearch/MappingPhase.java | 22 +++++++------
4 files changed, 39 insertions(+), 51 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
index 32c44671d0..131ca70e09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
@@ -61,6 +61,10 @@ public class EsResource extends Resource {
public static final String HTTP_SSL_ENABLED = "http_ssl_enabled";
public static final String QUERY_DSL = "query_dsl";
+ public static final String DOC_VALUE_SCAN_DEFAULT_VALUE = "true";
+ public static final String KEYWORD_SNIFF_DEFAULT_VALUE = "true";
+ public static final String HTTP_SSL_ENABLED_DEFAULT_VALUE = "false";
+ public static final String NODES_DISCOVERY_DEFAULT_VALUE = "true";
@SerializedName(value = "properties")
private Map<String, String> properties;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index 54a294e1aa..0c89a3b0d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -50,9 +50,9 @@ import java.util.Set;
@Getter
@Setter
public class EsTable extends Table {
- private static final Logger LOG = LogManager.getLogger(EsTable.class);
-
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new
HashSet<>(Arrays.asList("text"));
+
+ private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:
// It is possible to notice that retrieving an high number of fields leads
// to a sensible worsening of performance if DocValues are used.
@@ -80,16 +80,16 @@ public class EsTable extends Table {
private EsTablePartitions esTablePartitions;
// Whether to enable docvalues scan optimization for fetching fields more
fast, default to true
- private boolean enableDocValueScan = true;
+ private boolean enableDocValueScan =
Boolean.parseBoolean(EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE);
// Whether to enable sniffing keyword for filtering more reasonable,
default to true
- private boolean enableKeywordSniff = true;
+ private boolean enableKeywordSniff =
Boolean.parseBoolean(EsResource.KEYWORD_SNIFF_DEFAULT_VALUE);
// if the number of fields which value extracted from `doc_value`
exceeding this max limitation
// would downgrade to extract value from `stored_fields`
private int maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
- private boolean nodesDiscovery = true;
+ private boolean nodesDiscovery =
Boolean.parseBoolean(EsResource.NODES_DISCOVERY_DEFAULT_VALUE);
- private boolean httpSslEnabled = false;
+ private boolean httpSslEnabled =
Boolean.parseBoolean(EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE);
// tableContext is used for being convenient to persist some configuration
parameters uniformly
private Map<String, String> tableContext = new HashMap<>();
@@ -260,12 +260,10 @@ public class EsTable extends Table {
indexName = tableContext.get("indexName");
mappingType = tableContext.get("mappingType");
- enableDocValueScan =
Boolean.parseBoolean(tableContext.get("enableDocValueScan"));
- if (tableContext.containsKey("enableKeywordSniff")) {
- enableKeywordSniff =
Boolean.parseBoolean(tableContext.get("enableKeywordSniff"));
- } else {
- enableKeywordSniff = true;
- }
+ enableDocValueScan =
Boolean.parseBoolean(tableContext.getOrDefault("enableDocValueScan",
+ EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
+ enableKeywordSniff =
Boolean.parseBoolean(tableContext.getOrDefault("enableKeywordSniff",
+ EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
if (tableContext.containsKey("maxDocValueFields")) {
try {
maxDocValueFields =
Integer.parseInt(tableContext.get("maxDocValueFields"));
@@ -273,16 +271,10 @@ public class EsTable extends Table {
maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
}
}
- if (tableContext.containsKey(EsResource.NODES_DISCOVERY)) {
- nodesDiscovery =
Boolean.parseBoolean(tableContext.get(EsResource.NODES_DISCOVERY));
- } else {
- nodesDiscovery = true;
- }
- if (tableContext.containsKey(EsResource.HTTP_SSL_ENABLED)) {
- httpSslEnabled =
Boolean.parseBoolean(tableContext.get(EsResource.HTTP_SSL_ENABLED));
- } else {
- httpSslEnabled = false;
- }
+ nodesDiscovery =
Boolean.parseBoolean(tableContext.getOrDefault(EsResource.NODES_DISCOVERY,
+ EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
+ httpSslEnabled =
Boolean.parseBoolean(tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED,
+ EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
PartitionType partType = PartitionType.valueOf(Text.readString(in));
if (partType == PartitionType.UNPARTITIONED) {
partitionInfo = SinglePartitionInfo.read(in);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
index 455ea0cb5c..f3441e0c9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
@@ -17,7 +17,6 @@
package org.apache.doris.datasource;
-
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EsResource;
@@ -41,10 +40,9 @@ import java.util.Map;
*/
@Getter
public class EsExternalCatalog extends ExternalCatalog {
- private static final Logger LOG =
LogManager.getLogger(EsExternalCatalog.class);
-
public static final String DEFAULT_DB = "default_db";
+ private static final Logger LOG =
LogManager.getLogger(EsExternalCatalog.class);
private EsRestClient esRestClient;
/**
@@ -66,27 +64,15 @@ public class EsExternalCatalog extends ExternalCatalog {
if (properties.containsKey("ssl")) {
properties.put(EsResource.HTTP_SSL_ENABLED,
properties.remove("ssl"));
}
- if (!properties.containsKey(EsResource.HTTP_SSL_ENABLED)) {
- properties.put(EsResource.HTTP_SSL_ENABLED, String.valueOf(false));
- }
if (properties.containsKey("username")) {
properties.put(EsResource.USER, properties.remove("username"));
}
if (properties.containsKey("doc_value_scan")) {
properties.put(EsResource.DOC_VALUE_SCAN,
properties.remove("doc_value_scan"));
}
- if (!properties.containsKey(EsResource.DOC_VALUE_SCAN)) {
- properties.put(EsResource.DOC_VALUE_SCAN, "true");
- }
if (properties.containsKey("keyword_sniff")) {
properties.put(EsResource.KEYWORD_SNIFF,
properties.remove("keyword_sniff"));
}
- if (!properties.containsKey(EsResource.KEYWORD_SNIFF)) {
- properties.put(EsResource.KEYWORD_SNIFF, "true");
- }
- if (!properties.containsKey(EsResource.NODES_DISCOVERY)) {
- properties.put(EsResource.NODES_DISCOVERY, "true");
- }
return properties;
}
@@ -104,19 +90,23 @@ public class EsExternalCatalog extends ExternalCatalog {
}
public boolean enableDocValueScan() {
- return
Boolean.valueOf(catalogProperty.getOrDefault(EsResource.DOC_VALUE_SCAN,
"true"));
+ return
Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.DOC_VALUE_SCAN,
+ EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
}
public boolean enableKeywordSniff() {
- return
Boolean.valueOf(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF, "true"));
+ return
Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF,
+ EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
}
public boolean enableSsl() {
- return
Boolean.valueOf(catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED,
"false"));
+ return
Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED,
+ EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
}
public boolean enableNodesDiscovery() {
- return
Boolean.valueOf(catalogProperty.getOrDefault(EsResource.NODES_DISCOVERY,
"true"));
+ return
Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.NODES_DISCOVERY,
+ EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index 493fa413a3..42500d7f96 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -58,15 +58,17 @@ public class MappingPhase implements SearchPhase {
JSONObject properties =
EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping,
searchContext.type());
for (Column col : searchContext.columns()) {
String colName = col.getName();
- // if column exists in Doris Table but no found in ES's mapping,
we choose to ignore this situation?
- if (!properties.containsKey(colName)) {
- throw new DorisEsException(
- "index[" + searchContext.sourceIndex() + "] type[" +
indexMapping + "] mapping not found column"
- + colName + " for the ES Cluster");
+ // _id not exist mapping, but be can query it.
+ if (!"_id".equals(colName)) {
+ if (!properties.containsKey(colName)) {
+ throw new DorisEsException(
+ "index[" + searchContext.sourceIndex() + "]
mapping[" + indexMapping + "] not found "
+ + "column " + colName + " for the ES
Cluster");
+ }
+ JSONObject fieldObject = (JSONObject) properties.get(colName);
+ resolveKeywordFields(searchContext, fieldObject, colName);
+ resolveDocValuesFields(searchContext, fieldObject, colName);
}
- JSONObject fieldObject = (JSONObject) properties.get(colName);
- resolveKeywordFields(searchContext, fieldObject, colName);
- resolveDocValuesFields(searchContext, fieldObject, colName);
}
}
@@ -80,9 +82,9 @@ public class MappingPhase implements SearchPhase {
JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
if (fieldsObject != null) {
for (Object key : fieldsObject.keySet()) {
- JSONObject innerTypeObject = (JSONObject)
fieldsObject.get((String) key);
+ JSONObject innerTypeObject = (JSONObject)
fieldsObject.get(key);
// just for text type
- if ("keyword".equals((String)
innerTypeObject.get("type"))) {
+ if ("keyword".equals(innerTypeObject.get("type"))) {
searchContext.fetchFieldsContext().put(colName,
colName + "." + key);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
