This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 34d95cef1fc [improve](ES Catalog)Only push down literal expr in binary
predicate(#39845) (#39847)
34d95cef1fc is described below
commit 34d95cef1fcfd352d595332ed8c52ef7c2095dd5
Author: qiye <jianliang5...@gmail.com>
AuthorDate: Thu Aug 29 14:07:17 2024 +0800
[improve](ES Catalog)Only push down literal expr in binary
predicate(#39845) (#39847)
## Proposed changes
bp #39845
---
.../apache/doris/datasource/es/QueryBuilders.java | 77 +++++++++++++++++-----
.../external/elasticsearch/QueryBuildersTest.java | 27 ++++++++
2 files changed, 86 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java
index 3a54e012a32..19930bb2b14 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java
@@ -31,6 +31,7 @@ import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.LikePredicate.Operator;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.EsResource;
import org.apache.doris.thrift.TExprOpcode;
@@ -127,9 +128,24 @@ public final class QueryBuilders {
.build());
}
- private static QueryBuilder parseBinaryPredicate(Expr expr, TExprOpcode
opCode, String column,
+ private static TExprOpcode flipOpCode(TExprOpcode opCode) {
+ switch (opCode) {
+ case GE:
+ return TExprOpcode.LE;
+ case GT:
+ return TExprOpcode.LT;
+ case LE:
+ return TExprOpcode.GE;
+ case LT:
+ return TExprOpcode.GT;
+ default:
+ return opCode;
+ }
+ }
+
+ private static QueryBuilder parseBinaryPredicate(LiteralExpr expr,
TExprOpcode opCode, String column,
boolean needDateCompat) {
- Object value = toDorisLiteral(expr.getChild(1));
+ Object value = toDorisLiteral(expr);
if (needDateCompat) {
value = compatDefaultDate(value);
}
@@ -223,6 +239,20 @@ public final class QueryBuilders {
return new QueryBuilders.EsQueryBuilder(stringValue);
}
+ private static String getColumnFromExpr(Expr expr) {
+ // Type transformed cast can not pushdown
+ if (expr instanceof CastExpr) {
+ Expr withoutCastExpr = exprWithoutCast(expr);
+ if (withoutCastExpr.getType().equals(expr.getType())
+ || (withoutCastExpr.getType().isFloatingPointType() &&
expr.getType().isFloatingPointType())) {
+ return ((SlotRef) withoutCastExpr).getColumnName();
+ }
+ } else if (expr instanceof SlotRef) {
+ return ((SlotRef) expr).getColumnName();
+ }
+ return null;
+ }
+
/**
* Doris expr to es dsl.
**/
@@ -241,32 +271,43 @@ public final class QueryBuilders {
return toCompoundEsDsl(expr, notPushDownList, fieldsContext,
builderOptions);
}
TExprOpcode opCode = expr.getOpcode();
- String column;
+ boolean isFlip = false;
Expr leftExpr = expr.getChild(0);
- // Type transformed cast can not pushdown
- if (leftExpr instanceof CastExpr) {
- Expr withoutCastExpr = exprWithoutCast(leftExpr);
- // pushdown col(float) >= 3
- if (withoutCastExpr.getType().equals(leftExpr.getType()) ||
(withoutCastExpr.getType().isFloatingPointType()
- && leftExpr.getType().isFloatingPointType())) {
- column = ((SlotRef) withoutCastExpr).getColumnName();
- } else {
- notPushDownList.add(expr);
- return null;
- }
- } else if (leftExpr instanceof SlotRef) {
- column = ((SlotRef) leftExpr).getColumnName();
- } else {
+ String column = getColumnFromExpr(leftExpr);
+
+ if (StringUtils.isEmpty(column)) {
+ Expr rightExpr = expr.getChild(1);
+ column = getColumnFromExpr(rightExpr);
+ opCode = flipOpCode(opCode);
+ isFlip = true;
+ }
+
+ if (StringUtils.isEmpty(column)) {
notPushDownList.add(expr);
return null;
}
+
// Check whether the date type need compat, it must before keyword
replace.
List<String> needCompatDateFields =
builderOptions.getNeedCompatDateFields();
boolean needDateCompat = needCompatDateFields != null &&
needCompatDateFields.contains(column);
// Replace col with col.keyword if mapping exist.
column = fieldsContext.getOrDefault(column, column);
if (expr instanceof BinaryPredicate) {
- return parseBinaryPredicate(expr, opCode, column, needDateCompat);
+ BinaryPredicate binaryPredicate = (BinaryPredicate) expr;
+ Expr value;
+ if (isFlip) {
+ value = binaryPredicate.getChild(0);
+ } else {
+ value = binaryPredicate.getChild(1);
+ }
+ // only push down literal expr to ES
+ if (value instanceof LiteralExpr) {
+ LiteralExpr literalExpr = (LiteralExpr) value;
+ return parseBinaryPredicate(literalExpr, opCode, column,
needDateCompat);
+ } else {
+ notPushDownList.add(expr);
+ return null;
+ }
}
if (expr instanceof IsNullPredicate) {
return parseIsNullPredicate(expr, column);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java
index 3cf9261b932..ca5344990e7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java
@@ -209,6 +209,33 @@ public class QueryBuildersTest {
new FloatLiteral(3.0, Type.DOUBLE));
QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList,
fieldsContext, builderOptions);
Assertions.assertEquals(3, notPushDownList.size());
+
+ SlotRef k4 = new SlotRef(null, "k4");
+ k4.setType(Type.FLOAT);
+ CastExpr castFloatExpr = new CastExpr(Type.FLOAT, k4);
+ BinaryPredicate castFloatPredicate = new BinaryPredicate(Operator.GE,
new FloatLiteral(3.0, Type.FLOAT),
+ castFloatExpr);
+ QueryBuilders.QueryBuilder queryBuilder =
QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext,
builderOptions);
+ Assertions.assertEquals("{\"range\":{\"k4\":{\"lte\":3.0}}}",
queryBuilder.toJson());
+ Assertions.assertEquals(3, notPushDownList.size());
+
+ castFloatPredicate = new BinaryPredicate(Operator.LE, new
FloatLiteral(3.0, Type.FLOAT),
+ castFloatExpr);
+ queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate,
notPushDownList, fieldsContext, builderOptions);
+ Assertions.assertEquals("{\"range\":{\"k4\":{\"gte\":3.0}}}",
queryBuilder.toJson());
+ Assertions.assertEquals(3, notPushDownList.size());
+
+ castFloatPredicate = new BinaryPredicate(Operator.LT, new
FloatLiteral(3.0, Type.FLOAT),
+ castFloatExpr);
+ queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate,
notPushDownList, fieldsContext, builderOptions);
+ Assertions.assertEquals("{\"range\":{\"k4\":{\"gt\":3.0}}}",
queryBuilder.toJson());
+ Assertions.assertEquals(3, notPushDownList.size());
+
+ castFloatPredicate = new BinaryPredicate(Operator.GT, new
FloatLiteral(3.0, Type.FLOAT),
+ castFloatExpr);
+ queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate,
notPushDownList, fieldsContext, builderOptions);
+ Assertions.assertEquals("{\"range\":{\"k4\":{\"lt\":3.0}}}",
queryBuilder.toJson());
+ Assertions.assertEquals(3, notPushDownList.size());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
