Jackie-Jiang commented on code in PR #8518: URL: https://github.com/apache/pinot/pull/8518#discussion_r861402444
########## pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java: ########## @@ -163,94 +165,113 @@ public static FunctionContext getFunction(FunctionCallAstNode astNode) { /** * Converts the given Thrift {@link Expression} into a {@link FilterContext}. * <p>NOTE: Currently the query engine only accepts string literals as the right-hand side of the predicate, so we - * always convert the right-hand side expressions into strings. + * always convert the right-hand side expressions into strings. We also update boolean predicates that are + * missing an EQUALS filter operator. */ public static FilterContext getFilter(Expression thriftExpression) { - Function thriftFunction = thriftExpression.getFunctionCall(); - FilterKind filterKind = FilterKind.valueOf(thriftFunction.getOperator().toUpperCase()); - List<Expression> operands = thriftFunction.getOperands(); - int numOperands = operands.size(); - switch (filterKind) { - case AND: - List<FilterContext> children = new ArrayList<>(numOperands); - for (Expression operand : operands) { - children.add(getFilter(operand)); - } - return new FilterContext(FilterContext.Type.AND, children, null); - case OR: - children = new ArrayList<>(numOperands); - for (Expression operand : operands) { - children.add(getFilter(operand)); - } - return new FilterContext(FilterContext.Type.OR, children, null); - case NOT: - assert numOperands == 1; - return new FilterContext(FilterContext.Type.NOT, new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); - case EQUALS: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new EqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case NOT_EQUALS: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new NotEqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case IN: - List<String> values = new ArrayList<>(numOperands - 1); - for (int i = 1; i < numOperands; i++) { - values.add(getStringValue(operands.get(i))); - } - return new FilterContext(FilterContext.Type.PREDICATE, null, - new InPredicate(getExpression(operands.get(0)), values)); - case NOT_IN: - values = new ArrayList<>(numOperands - 1); - for (int i = 1; i < numOperands; i++) { - values.add(getStringValue(operands.get(i))); - } - return new FilterContext(FilterContext.Type.PREDICATE, null, - new NotInPredicate(getExpression(operands.get(0)), values)); - case GREATER_THAN: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), false, getStringValue(operands.get(1)), false, - RangePredicate.UNBOUNDED)); - case GREATER_THAN_OR_EQUAL: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), true, getStringValue(operands.get(1)), false, - RangePredicate.UNBOUNDED)); - case LESS_THAN: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), false, RangePredicate.UNBOUNDED, false, - getStringValue(operands.get(1)))); - case LESS_THAN_OR_EQUAL: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), false, RangePredicate.UNBOUNDED, true, - getStringValue(operands.get(1)))); - case BETWEEN: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), true, getStringValue(operands.get(1)), true, - getStringValue(operands.get(2)))); - case RANGE: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case REGEXP_LIKE: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RegexpLikePredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case LIKE: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RegexpLikePredicate(getExpression(operands.get(0)), - RegexpPatternConverterUtils.likeToRegexpLike(getStringValue(operands.get(1))))); - case TEXT_MATCH: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new TextMatchPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case JSON_MATCH: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new JsonMatchPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case IS_NULL: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new IsNullPredicate(getExpression(operands.get(0)))); - case IS_NOT_NULL: + ExpressionType type = thriftExpression.getType(); Review Comment: Suggest refactoring the code a little bit so that it does not show as refactoring the whole thing: - Add `private static FilterContext getFilter(Function thriftFunction)` - Do a `switch(type)` and call it if it is `FUNCTION` type ########## pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java: ########## @@ -19,33 +19,41 @@ package org.apache.pinot.pql.parsers.pql2.ast; public enum FilterKind { - AND, - OR, - NOT, - EQUALS, - NOT_EQUALS, - GREATER_THAN, - GREATER_THAN_OR_EQUAL, - LESS_THAN, - LESS_THAN_OR_EQUAL, - LIKE, - BETWEEN, - RANGE, - IN, - NOT_IN, - REGEXP_LIKE, - IS_NULL, - IS_NOT_NULL, - TEXT_MATCH, - JSON_MATCH; + AND(false, false), Review Comment: Is this change related? We should not rely on type `FilterKind` for the purpose of this PR ########## pinot-common/src/main/java/org/apache/pinot/pql/parsers/pql2/ast/FilterKind.java: ########## @@ -19,33 +19,41 @@ package org.apache.pinot.pql.parsers.pql2.ast; public enum FilterKind { - AND, - OR, - NOT, - EQUALS, - NOT_EQUALS, - GREATER_THAN, - GREATER_THAN_OR_EQUAL, - LESS_THAN, - LESS_THAN_OR_EQUAL, - LIKE, - BETWEEN, - RANGE, - IN, - NOT_IN, - REGEXP_LIKE, - IS_NULL, - IS_NOT_NULL, - TEXT_MATCH, - JSON_MATCH; + AND(false, false), + OR(false, false), + NOT(false, false), + EQUALS(true, false), + NOT_EQUALS(true, false), + GREATER_THAN(true, true), + GREATER_THAN_OR_EQUAL(true, true), + LESS_THAN(true, true), + LESS_THAN_OR_EQUAL(true, true), + LIKE(true, false), + BETWEEN(true, true), + RANGE(true, true), + IN(true, false), + NOT_IN(true, false), + REGEXP_LIKE(true, false), + IS_NULL(true, false), + IS_NOT_NULL(true, false), + TEXT_MATCH(true, false), + JSON_MATCH(true, false); - /** - * Helper method that returns true if the enum maps to a Range. - * - * @return True if the enum is of Range type, false otherwise. - */ + private final boolean _isPredicate; + private final boolean _isRange; + + FilterKind(boolean isPredicate, boolean isRange) { + _isPredicate = isPredicate; + _isRange = isRange; + } + + /** @return true if the enum is of Range type, false otherwise. */ public boolean isRange() { - return this == GREATER_THAN || this == GREATER_THAN_OR_EQUAL || this == LESS_THAN || this == LESS_THAN_OR_EQUAL - || this == BETWEEN || this == RANGE; + return _isRange; + } + + /** @return true if the filter is a predicate. This logic should mimic FilterContext.Type. */ + public boolean isPredicate() { Review Comment: Don't add this API for now if it is not used ########## pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java: ########## @@ -163,94 +165,113 @@ public static FunctionContext getFunction(FunctionCallAstNode astNode) { /** * Converts the given Thrift {@link Expression} into a {@link FilterContext}. * <p>NOTE: Currently the query engine only accepts string literals as the right-hand side of the predicate, so we - * always convert the right-hand side expressions into strings. + * always convert the right-hand side expressions into strings. We also update boolean predicates that are + * missing an EQUALS filter operator. */ public static FilterContext getFilter(Expression thriftExpression) { - Function thriftFunction = thriftExpression.getFunctionCall(); - FilterKind filterKind = FilterKind.valueOf(thriftFunction.getOperator().toUpperCase()); - List<Expression> operands = thriftFunction.getOperands(); - int numOperands = operands.size(); - switch (filterKind) { - case AND: - List<FilterContext> children = new ArrayList<>(numOperands); - for (Expression operand : operands) { - children.add(getFilter(operand)); - } - return new FilterContext(FilterContext.Type.AND, children, null); - case OR: - children = new ArrayList<>(numOperands); - for (Expression operand : operands) { - children.add(getFilter(operand)); - } - return new FilterContext(FilterContext.Type.OR, children, null); - case NOT: - assert numOperands == 1; - return new FilterContext(FilterContext.Type.NOT, new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); - case EQUALS: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new EqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case NOT_EQUALS: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new NotEqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case IN: - List<String> values = new ArrayList<>(numOperands - 1); - for (int i = 1; i < numOperands; i++) { - values.add(getStringValue(operands.get(i))); - } - return new FilterContext(FilterContext.Type.PREDICATE, null, - new InPredicate(getExpression(operands.get(0)), values)); - case NOT_IN: - values = new ArrayList<>(numOperands - 1); - for (int i = 1; i < numOperands; i++) { - values.add(getStringValue(operands.get(i))); - } - return new FilterContext(FilterContext.Type.PREDICATE, null, - new NotInPredicate(getExpression(operands.get(0)), values)); - case GREATER_THAN: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), false, getStringValue(operands.get(1)), false, - RangePredicate.UNBOUNDED)); - case GREATER_THAN_OR_EQUAL: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), true, getStringValue(operands.get(1)), false, - RangePredicate.UNBOUNDED)); - case LESS_THAN: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), false, RangePredicate.UNBOUNDED, false, - getStringValue(operands.get(1)))); - case LESS_THAN_OR_EQUAL: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), false, RangePredicate.UNBOUNDED, true, - getStringValue(operands.get(1)))); - case BETWEEN: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), true, getStringValue(operands.get(1)), true, - getStringValue(operands.get(2)))); - case RANGE: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RangePredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case REGEXP_LIKE: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RegexpLikePredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case LIKE: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new RegexpLikePredicate(getExpression(operands.get(0)), - RegexpPatternConverterUtils.likeToRegexpLike(getStringValue(operands.get(1))))); - case TEXT_MATCH: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new TextMatchPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case JSON_MATCH: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new JsonMatchPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); - case IS_NULL: - return new FilterContext(FilterContext.Type.PREDICATE, null, - new IsNullPredicate(getExpression(operands.get(0)))); - case IS_NOT_NULL: + ExpressionType type = thriftExpression.getType(); + if (type == ExpressionType.FUNCTION) { + Function thriftFunction = thriftExpression.getFunctionCall(); + String functionOperator = thriftFunction.getOperator(); + + // convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str') = true" + if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) { return new FilterContext(FilterContext.Type.PREDICATE, null, - new IsNotNullPredicate(getExpression(operands.get(0)))); - default: - throw new IllegalStateException(); + new EqPredicate(getExpression(thriftExpression), getStringValue(RequestUtils.getLiteralExpression(true)))); + } + + FilterKind filterKind = FilterKind.valueOf(thriftFunction.getOperator().toUpperCase()); + List<Expression> operands = thriftFunction.getOperands(); + int numOperands = operands.size(); + switch (filterKind) { + case AND: + List<FilterContext> children = new ArrayList<>(numOperands); + for (Expression operand : operands) { + children.add(getFilter(operand)); + } + return new FilterContext(FilterContext.Type.AND, children, null); + case OR: + children = new ArrayList<>(numOperands); + for (Expression operand : operands) { + children.add(getFilter(operand)); + } + return new FilterContext(FilterContext.Type.OR, children, null); + case NOT: + assert numOperands == 1; + return new FilterContext(FilterContext.Type.NOT, + new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); + case EQUALS: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new EqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); + case NOT_EQUALS: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new NotEqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); + case IN: + List<String> values = new ArrayList<>(numOperands - 1); + for (int i = 1; i < numOperands; i++) { + values.add(getStringValue(operands.get(i))); + } + return new FilterContext(FilterContext.Type.PREDICATE, null, + new InPredicate(getExpression(operands.get(0)), values)); + case NOT_IN: + values = new ArrayList<>(numOperands - 1); + for (int i = 1; i < numOperands; i++) { + values.add(getStringValue(operands.get(i))); + } + return new FilterContext(FilterContext.Type.PREDICATE, null, + new NotInPredicate(getExpression(operands.get(0)), values)); + case GREATER_THAN: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RangePredicate(getExpression(operands.get(0)), false, getStringValue(operands.get(1)), false, + RangePredicate.UNBOUNDED)); + case GREATER_THAN_OR_EQUAL: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RangePredicate(getExpression(operands.get(0)), true, getStringValue(operands.get(1)), false, + RangePredicate.UNBOUNDED)); + case LESS_THAN: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RangePredicate(getExpression(operands.get(0)), false, RangePredicate.UNBOUNDED, false, + getStringValue(operands.get(1)))); + case LESS_THAN_OR_EQUAL: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RangePredicate(getExpression(operands.get(0)), false, RangePredicate.UNBOUNDED, true, + getStringValue(operands.get(1)))); + case BETWEEN: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RangePredicate(getExpression(operands.get(0)), true, getStringValue(operands.get(1)), true, + getStringValue(operands.get(2)))); + case RANGE: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RangePredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); + case REGEXP_LIKE: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RegexpLikePredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); + case LIKE: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new RegexpLikePredicate(getExpression(operands.get(0)), + RegexpPatternConverterUtils.likeToRegexpLike(getStringValue(operands.get(1))))); + case TEXT_MATCH: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new TextMatchPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); + case JSON_MATCH: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new JsonMatchPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); + case IS_NULL: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new IsNullPredicate(getExpression(operands.get(0)))); + case IS_NOT_NULL: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new IsNotNullPredicate(getExpression(operands.get(0)))); + default: + throw new IllegalStateException(); + } + } else if (type == ExpressionType.IDENTIFIER) { + // Convert "WHERE a" to "WHERE a = true" + return new FilterContext(FilterContext.Type.PREDICATE, null, + new EqPredicate(getExpression(thriftExpression), getStringValue(RequestUtils.getLiteralExpression(true)))); } + + throw new IllegalArgumentException(); Review Comment: We should also handle `literal`, where we handle `false`, `0`, `0.0` as false, others as true. Currently literal is always represented as string, so we have to take this work-around. Once we preserve the literal type, this can be simplified ########## pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java: ########## @@ -33,75 +36,126 @@ public class PredicateComparisonRewriter implements QueryRewriter { public PinotQuery rewrite(PinotQuery pinotQuery) { Expression filterExpression = pinotQuery.getFilterExpression(); if (filterExpression != null) { - pinotQuery.setFilterExpression(updateComparisonPredicate(filterExpression)); + pinotQuery.setFilterExpression(updatePredicate(filterExpression, null)); } Expression havingExpression = pinotQuery.getHavingExpression(); if (havingExpression != null) { - pinotQuery.setHavingExpression(updateComparisonPredicate(havingExpression)); + pinotQuery.setHavingExpression(updatePredicate(havingExpression, null)); } return pinotQuery; } - // This method converts a predicate expression to the what Pinot could evaluate. - // For comparison expression, left operand could be any expression, but right operand only - // supports literal. - // E.g. 'WHERE a > b' will be updated to 'WHERE a - b > 0' - private static Expression updateComparisonPredicate(Expression expression) { - Function function = expression.getFunctionCall(); - if (function != null) { - FilterKind filterKind; - try { - filterKind = FilterKind.valueOf(function.getOperator()); - } catch (Exception e) { - throw new SqlCompilationException("Unsupported filter kind: " + function.getOperator()); - } - List<Expression> operands = function.getOperands(); - switch (filterKind) { - case AND: - case OR: - case NOT: - operands.replaceAll(PredicateComparisonRewriter::updateComparisonPredicate); - break; - case EQUALS: - case NOT_EQUALS: - case GREATER_THAN: - case GREATER_THAN_OR_EQUAL: - case LESS_THAN: - case LESS_THAN_OR_EQUAL: - Expression firstOperand = operands.get(0); - Expression secondOperand = operands.get(1); + /** + * This method converts a predicate expression to the what Pinot could evaluate. + * 1. For comparison expression, left operand could be any expression, but right operand only + * supports literal. E.g. 'WHERE a > b' will be converted to 'WHERE a - b > 0' + * 2. Updates boolean predicates (literals and scalar functions) that are missing an EQUALS filter. + * E.g. 1: 'WHERE a' will be updated to 'WHERE a = true' + * E.g. 2: "WHERE startsWith(col, 'str')" will be updated to "WHERE startsWith(col, 'str') = true" + * + * @param expression current expression in the expression tree + * @param parentFunction parentFunction parent expression + * @return re-written expression. + */ + private static Expression updatePredicate(Expression expression, Function parentFunction) { + ExpressionType type = expression.getType(); + if (type == ExpressionType.FUNCTION) { + Function function = expression.getFunctionCall(); + String functionOperator = function.getOperator(); - // Handle predicate like '10 = a' -> 'a = 10' - if (firstOperand.isSetLiteral()) { - if (!secondOperand.isSetLiteral()) { - function.setOperator(getOppositeOperator(filterKind).name()); - operands.set(0, secondOperand); - operands.set(1, firstOperand); + if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) { + // If the function is not of FilterKind, we might have to rewrite the function if parentFunction is not a + // predicate. + // Example: A query like "select col1 from table where startsWith(col1, 'myStr') AND col2 > 10;" should be + // rewritten to "select col1 from table where startsWith(col1, 'myStr') = true AND col2 > 10;". + expression = convertPredicateToBooleanExpression(expression, parentFunction); + return expression; + } else { + FilterKind filterKind = FilterKind.valueOf(function.getOperator()); + List<Expression> operands = function.getOperands(); + switch (filterKind) { + case AND: + case OR: + case NOT: + for (int i = 0; i < operands.size(); i++) { + Expression operand = operands.get(i); + operands.set(i, updatePredicate(operand, function)); } break; - } + case EQUALS: + case NOT_EQUALS: + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + Expression firstOperand = operands.get(0); + Expression secondOperand = operands.get(1); - // Handle predicate like 'a > b' -> 'a - b > 0' - if (!secondOperand.isSetLiteral()) { - Expression minusExpression = RequestUtils.getFunctionExpression("minus"); - minusExpression.getFunctionCall().setOperands(Arrays.asList(firstOperand, secondOperand)); - operands.set(0, minusExpression); - operands.set(1, RequestUtils.getLiteralExpression(0)); + // Handle predicate like '10 = a' -> 'a = 10' + if (firstOperand.isSetLiteral()) { + if (!secondOperand.isSetLiteral()) { + function.setOperator(getOppositeOperator(filterKind).name()); + operands.set(0, secondOperand); + operands.set(1, firstOperand); + } + break; + } + + // Handle predicate like 'a > b' -> 'a - b > 0' + if (!secondOperand.isSetLiteral()) { + Expression minusExpression = RequestUtils.getFunctionExpression("minus"); + minusExpression.getFunctionCall().setOperands(Arrays.asList(firstOperand, secondOperand)); + operands.set(0, minusExpression); + operands.set(1, RequestUtils.getLiteralExpression(0)); + break; + } break; - } - break; - default: - int numOperands = operands.size(); - for (int i = 1; i < numOperands; i++) { - if (!operands.get(i).isSetLiteral()) { - throw new SqlCompilationException( - String.format("For %s predicate, the operands except for the first one must be literal, got: %s", - filterKind, expression)); + default: + int numOperands = operands.size(); + for (int i = 1; i < numOperands; i++) { + if (!operands.get(i).isSetLiteral()) { + throw new SqlCompilationException(String + .format("For %s predicate, the operands except for the first one must be literal, got: %s", + filterKind, expression)); + } } - } - break; + break; + } } + } else if (type == ExpressionType.IDENTIFIER) { + expression = convertPredicateToBooleanExpression(expression, parentFunction); } + Review Comment: We should also update literal to match the SQL semantic -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org