airborne12 commented on code in PR #69292: URL: https://github.com/apache/doris/pull/69292#discussion_r4335465636
########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BindScoreToOlapScan.java: ########## @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Score; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalApply; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalView; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Bind score() to the olap scan that computes it. + * + * <p>score() is the relevance of a row for the MATCH and SEARCH predicates in the WHERE clause of + * its SELECT, and only the olap scan that evaluates those predicates computes it. So score() is + * supported in a SELECT that reads one olap table and has a MATCH or SEARCH predicate in its WHERE + * clause: + * <ul> + * <li>in the WHERE clause, as one minimum score: score() > x or score() >= x;</li> + * <li>in the select list, window functions and ORDER BY of a SELECT that does not aggregate.</li> + * </ul> + * A SELECT in a subquery, view, CTE or set operation has to ORDER BY score() with a LIMIT, which keeps + * the predicates of the outer query away from its scan. Any other use of score() is rejected. + * + * <p>This rule runs before subquery aliases are removed. It adds a score column to the scan, which + * scores every row the scan returns, and replaces the score() calls of the SELECT with it. + * PushDownScoreRangeIntoOlapScan and PushDownScoreTopNIntoOlapScan push the minimum score and a + * TopN on the score into the scan. + */ +public class BindScoreToOlapScan implements CustomRewriter { + public static final String UNSUPPORTED_SELECT_ERROR = "score() can only be used in a SELECT that reads one " + + "table and has a MATCH or SEARCH predicate in its WHERE clause"; + public static final String NESTED_SELECT_ERROR = "score() in a subquery, view, CTE or set operation " + + "requires ORDER BY score() with LIMIT"; + public static final String AGGREGATE_ERROR = "score() can only be used in the WHERE clause of a SELECT " + + "with GROUP BY, aggregate functions or DISTINCT"; + public static final String HAVING_OR_QUALIFY_ERROR = "score(), MATCH and SEARCH cannot be used in HAVING " + + "or QUALIFY of a SELECT that calls score()"; + public static final String ONE_RANGE_ERROR = "Only one score() range predicate is supported in WHERE clause"; + public static final String RANGE_FORM_ERROR = "score() predicate in WHERE clause must be in the form of " + + "'score() > literal' or 'score() >= literal'"; + public static final String TOP_LEVEL_RANGE_ERROR = "score() predicate must be a top-level AND condition " + + "in WHERE clause"; + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!plan.anyMatch(node -> callsScore((Plan) node))) { + return plan; + } + return bindSelect(plan, true); + } + + /** The score column that this rule added to the scan. */ + public static Optional<Slot> findScoreSlot(LogicalOlapScan scan) { + return scan.getVirtualColumns().stream() + .filter(column -> column instanceof Alias && ((Alias) column).child() instanceof Score) + .map(NamedExpression::toSlot) + .findFirst(); + } + + private Plan bindSelect(Plan root, boolean outermost) { + Plan plan = bindNestedSelects(root); + if (!selectCallsScore(plan)) { + return plan; + } + List<Plan> path = pathToScan(plan); + if (path.isEmpty()) { + throw new AnalysisException(UNSUPPORTED_SELECT_ERROR); + } + check(path, outermost); + return bind(path); + } + + private Plan bindNestedSelects(Plan plan) { + ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(plan.arity()); + boolean changed = false; + for (int i = 0; i < plan.arity(); i++) { + Plan child = plan.child(i); + Plan newChild = startsSelect(plan, i) ? bindSelect(child, false) : bindNestedSelects(child); + changed |= newChild != child; + newChildren.add(newChild); + } + return changed ? plan.withChildren(newChildren.build()) : plan; + } + + // Subqueries, views, CTEs and the inputs of set operations are SELECTs of their own, a table + // alias is not. + private static boolean startsSelect(Plan parent, int childIndex) { + if (parent instanceof LogicalSubQueryAlias) { + return !(parent.child(0) instanceof LogicalRelation); + } + return parent instanceof LogicalView || parent instanceof LogicalCTEProducer + || parent instanceof LogicalSetOperation || parent instanceof LogicalRecursiveUnionAnchor + || parent instanceof LogicalRecursiveUnionProducer + || parent instanceof LogicalCTEAnchor && childIndex == 0 + || parent instanceof LogicalApply && childIndex == 1; + } + + private static boolean selectCallsScore(Plan plan) { + if (callsScore(plan)) { + return true; + } + for (int i = 0; i < plan.arity(); i++) { + if (!startsSelect(plan, i) && selectCallsScore(plan.child(i))) { + return true; + } + } + return false; + } + + private static boolean callsScore(Plan plan) { + return plan.getExpressions().stream().anyMatch(expression -> expression.containsType(Score.class)); + } + + // The plan nodes from the root of the SELECT down to its olap scan; empty when the SELECT reads + // anything else than one olap table, e.g. a join, a subquery or a lateral view. + private static List<Plan> pathToScan(Plan root) { + List<Plan> path = new ArrayList<>(); + Plan node = root; + while (!(node instanceof LogicalOlapScan)) { + path.add(node); + if (node instanceof LogicalCTEAnchor) { + node = node.child(1); + } else if (node instanceof LogicalSink || node instanceof LogicalLimit || node instanceof LogicalSort + || node instanceof LogicalProject || node instanceof LogicalFilter + || node instanceof LogicalAggregate + || node instanceof LogicalSubQueryAlias && !startsSelect(node, 0)) { + node = node.child(0); + } else { + return ImmutableList.of(); + } + } + path.add(node); + return path; + } + + private static void check(List<Plan> path, boolean outermost) { + // The WHERE clause is the filters right above the scan and its table alias. + int clause = path.size() - 2; + if (path.get(clause) instanceof LogicalSubQueryAlias) { + clause--; + } + List<Expression> where = new ArrayList<>(); + for (; clause >= 0 && path.get(clause) instanceof LogicalFilter; clause--) { + where.addAll(((LogicalFilter<?>) path.get(clause)).getConjuncts()); + } + if (where.stream().noneMatch(conjunct -> conjunct.containsType(Match.class, Search.class))) { + throw new AnalysisException(UNSUPPORTED_SELECT_ERROR); + } + List<Expression> ranges = where.stream() + .filter(conjunct -> conjunct.containsType(Score.class)) + .collect(Collectors.toList()); + if (ranges.size() > 1) { + throw new AnalysisException(ONE_RANGE_ERROR); + } + ranges.forEach(BindScoreToOlapScan::checkRange); + + List<Plan> aboveWhere = path.subList(0, clause + 1); + boolean aggregates = aboveWhere.stream().anyMatch(LogicalAggregate.class::isInstance); + Set<Slot> scoreAliases = aboveWhere.stream() + .filter(LogicalProject.class::isInstance) + .flatMap(project -> ((LogicalProject<?>) project).getProjects().stream()) + .filter(project -> project.containsType(Score.class)) + .map(NamedExpression::toSlot) + .collect(ImmutableSet.toImmutableSet()); + for (Plan node : aboveWhere) { + if (node instanceof LogicalFilter && ((LogicalFilter<?>) node).getConjuncts().stream().anyMatch( + conjunct -> conjunct.containsType(Score.class, Match.class, Search.class) + || conjunct.getInputSlots().stream().anyMatch(scoreAliases::contains))) { + throw new AnalysisException(HAVING_OR_QUALIFY_ERROR); + } + if (aggregates && callsScore(node)) { + throw new AnalysisException(AGGREGATE_ERROR); + } + } + if (!outermost && !ordersByScoreWithLimit(aboveWhere)) { Review Comment: Fixed in 8fc7d7cf35. `BindScoreToOlapScan` now rejects score() anywhere in a correlated subquery with `score() cannot be used in a correlated subquery`. As you traced, `EliminateLimitUnderApply` and `EliminateSortUnderApply` remove the ORDER BY score() LIMIT that the nested-SELECT check relies on. The check covers the whole subquery, because a scored derived table inside it can end up directly under the Apply once the derived table is merged. Uncorrelated subqueries keep their LIMIT and stay supported. Your EXISTS query is now an error case in `test_score_filter_usage`. `BindScoreToOlapScanTest` also covers a scored derived table in a correlated EXISTS and a correlated scalar subquery; these tests fail with the previous binder. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BindScoreToOlapScan.java: ########## @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Score; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalApply; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalView; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Bind score() to the olap scan that computes it. + * + * <p>score() is the relevance of a row for the MATCH and SEARCH predicates in the WHERE clause of + * its SELECT, and only the olap scan that evaluates those predicates computes it. So score() is + * supported in a SELECT that reads one olap table and has a MATCH or SEARCH predicate in its WHERE + * clause: + * <ul> + * <li>in the WHERE clause, as one minimum score: score() > x or score() >= x;</li> + * <li>in the select list, window functions and ORDER BY of a SELECT that does not aggregate.</li> + * </ul> + * A SELECT in a subquery, view, CTE or set operation has to ORDER BY score() with a LIMIT, which keeps + * the predicates of the outer query away from its scan. Any other use of score() is rejected. + * + * <p>This rule runs before subquery aliases are removed. It adds a score column to the scan, which + * scores every row the scan returns, and replaces the score() calls of the SELECT with it. + * PushDownScoreRangeIntoOlapScan and PushDownScoreTopNIntoOlapScan push the minimum score and a + * TopN on the score into the scan. + */ +public class BindScoreToOlapScan implements CustomRewriter { + public static final String UNSUPPORTED_SELECT_ERROR = "score() can only be used in a SELECT that reads one " + + "table and has a MATCH or SEARCH predicate in its WHERE clause"; + public static final String NESTED_SELECT_ERROR = "score() in a subquery, view, CTE or set operation " + + "requires ORDER BY score() with LIMIT"; + public static final String AGGREGATE_ERROR = "score() can only be used in the WHERE clause of a SELECT " + + "with GROUP BY, aggregate functions or DISTINCT"; + public static final String HAVING_OR_QUALIFY_ERROR = "score(), MATCH and SEARCH cannot be used in HAVING " + + "or QUALIFY of a SELECT that calls score()"; + public static final String ONE_RANGE_ERROR = "Only one score() range predicate is supported in WHERE clause"; + public static final String RANGE_FORM_ERROR = "score() predicate in WHERE clause must be in the form of " + + "'score() > literal' or 'score() >= literal'"; + public static final String TOP_LEVEL_RANGE_ERROR = "score() predicate must be a top-level AND condition " + + "in WHERE clause"; + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!plan.anyMatch(node -> callsScore((Plan) node))) { + return plan; + } + return bindSelect(plan, true); + } + + /** The score column that this rule added to the scan. */ + public static Optional<Slot> findScoreSlot(LogicalOlapScan scan) { + return scan.getVirtualColumns().stream() + .filter(column -> column instanceof Alias && ((Alias) column).child() instanceof Score) + .map(NamedExpression::toSlot) + .findFirst(); + } + + private Plan bindSelect(Plan root, boolean outermost) { + Plan plan = bindNestedSelects(root); + if (!selectCallsScore(plan)) { + return plan; + } + List<Plan> path = pathToScan(plan); + if (path.isEmpty()) { + throw new AnalysisException(UNSUPPORTED_SELECT_ERROR); + } + check(path, outermost); + return bind(path); + } + + private Plan bindNestedSelects(Plan plan) { + ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(plan.arity()); + boolean changed = false; + for (int i = 0; i < plan.arity(); i++) { + Plan child = plan.child(i); + Plan newChild = startsSelect(plan, i) ? bindSelect(child, false) : bindNestedSelects(child); + changed |= newChild != child; + newChildren.add(newChild); + } + return changed ? plan.withChildren(newChildren.build()) : plan; + } + + // Subqueries, views, CTEs and the inputs of set operations are SELECTs of their own, a table + // alias is not. + private static boolean startsSelect(Plan parent, int childIndex) { + if (parent instanceof LogicalSubQueryAlias) { + return !(parent.child(0) instanceof LogicalRelation); + } + return parent instanceof LogicalView || parent instanceof LogicalCTEProducer + || parent instanceof LogicalSetOperation || parent instanceof LogicalRecursiveUnionAnchor + || parent instanceof LogicalRecursiveUnionProducer + || parent instanceof LogicalCTEAnchor && childIndex == 0 + || parent instanceof LogicalApply && childIndex == 1; + } + + private static boolean selectCallsScore(Plan plan) { + if (callsScore(plan)) { + return true; + } + for (int i = 0; i < plan.arity(); i++) { + if (!startsSelect(plan, i) && selectCallsScore(plan.child(i))) { + return true; + } + } + return false; + } + + private static boolean callsScore(Plan plan) { + return plan.getExpressions().stream().anyMatch(expression -> expression.containsType(Score.class)); + } + + // The plan nodes from the root of the SELECT down to its olap scan; empty when the SELECT reads + // anything else than one olap table, e.g. a join, a subquery or a lateral view. + private static List<Plan> pathToScan(Plan root) { + List<Plan> path = new ArrayList<>(); + Plan node = root; + while (!(node instanceof LogicalOlapScan)) { + path.add(node); + if (node instanceof LogicalCTEAnchor) { + node = node.child(1); + } else if (node instanceof LogicalSink || node instanceof LogicalLimit || node instanceof LogicalSort + || node instanceof LogicalProject || node instanceof LogicalFilter + || node instanceof LogicalAggregate + || node instanceof LogicalSubQueryAlias && !startsSelect(node, 0)) { + node = node.child(0); + } else { + return ImmutableList.of(); + } + } + path.add(node); + return path; + } + + private static void check(List<Plan> path, boolean outermost) { + // The WHERE clause is the filters right above the scan and its table alias. + int clause = path.size() - 2; + if (path.get(clause) instanceof LogicalSubQueryAlias) { + clause--; + } + List<Expression> where = new ArrayList<>(); + for (; clause >= 0 && path.get(clause) instanceof LogicalFilter; clause--) { + where.addAll(((LogicalFilter<?>) path.get(clause)).getConjuncts()); + } + if (where.stream().noneMatch(conjunct -> conjunct.containsType(Match.class, Search.class))) { + throw new AnalysisException(UNSUPPORTED_SELECT_ERROR); + } + List<Expression> ranges = where.stream() + .filter(conjunct -> conjunct.containsType(Score.class)) + .collect(Collectors.toList()); + if (ranges.size() > 1) { + throw new AnalysisException(ONE_RANGE_ERROR); + } + ranges.forEach(BindScoreToOlapScan::checkRange); + + List<Plan> aboveWhere = path.subList(0, clause + 1); + boolean aggregates = aboveWhere.stream().anyMatch(LogicalAggregate.class::isInstance); + Set<Slot> scoreAliases = aboveWhere.stream() + .filter(LogicalProject.class::isInstance) + .flatMap(project -> ((LogicalProject<?>) project).getProjects().stream()) + .filter(project -> project.containsType(Score.class)) + .map(NamedExpression::toSlot) + .collect(ImmutableSet.toImmutableSet()); + for (Plan node : aboveWhere) { + if (node instanceof LogicalFilter && ((LogicalFilter<?>) node).getConjuncts().stream().anyMatch( + conjunct -> conjunct.containsType(Score.class, Match.class, Search.class) + || conjunct.getInputSlots().stream().anyMatch(scoreAliases::contains))) { + throw new AnalysisException(HAVING_OR_QUALIFY_ERROR); + } + if (aggregates && callsScore(node)) { + throw new AnalysisException(AGGREGATE_ERROR); + } + } + if (!outermost && !ordersByScoreWithLimit(aboveWhere)) { + throw new AnalysisException(NESTED_SELECT_ERROR); + } + } + + // score() > x or score() >= x, also written x < score() or x <= score(). A cast of score() to + // FLOAT or DOUBLE keeps its value. + private static void checkRange(Expression conjunct) { + if (!(conjunct instanceof ComparisonPredicate)) { + throw new AnalysisException(TOP_LEVEL_RANGE_ERROR); + } + ComparisonPredicate comparison = (ComparisonPredicate) conjunct; + boolean scoreAbove = (comparison instanceof GreaterThan || comparison instanceof GreaterThanEqual) + && isScore(comparison.left()) && isConstant(comparison.right()); + boolean scoreBelow = (comparison instanceof LessThan || comparison instanceof LessThanEqual) + && isScore(comparison.right()) && isConstant(comparison.left()); + if (!scoreAbove && !scoreBelow) { + throw new AnalysisException(RANGE_FORM_ERROR); + } + } + + private static boolean isScore(Expression expression) { + if (expression instanceof Cast) { + return expression.getDataType().isFloatLikeType() && isScore(((Cast) expression).child()); + } + return expression instanceof Score; + } + + private static boolean isConstant(Expression expression) { + return expression.getInputSlots().isEmpty() && !expression.containsType(Score.class); + } + + // A TopN on the score above the SELECT, so that no outer predicate is pushed into the scan. + private static boolean ordersByScoreWithLimit(List<Plan> aboveWhere) { + for (int i = 0; i + 1 < aboveWhere.size(); i++) { + if (aboveWhere.get(i) instanceof LogicalLimit && aboveWhere.get(i + 1) instanceof LogicalSort) { Review Comment: Fixed in 8fc7d7cf35. The TopN check now skips projections of plain columns between LIMIT and ORDER BY, which is where analysis restores the select list. Your query is a positive case in `test_score_filter_usage` (`nested_score_topn_having` returns id 1 with its apple-only score 0.4700036) and in `BindScoreToOlapScanTest`, where it fails with the previous binder. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckScoreUsage.java: ########## @@ -18,98 +18,187 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.scalar.Score; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalApply; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalView; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import com.google.common.collect.ImmutableList; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Optional; /** - * Check score function usage in project and aggregate without proper optimization context. + * Tie every score() call to the MATCH and SEARCH predicates it scores. * - * This rule ensures that score() function is used only in contexts where it can be optimized, - * requiring WHERE clause with MATCH function, ORDER BY and LIMIT. + * <p>score() is the relevance of a row for the MATCH and SEARCH predicates in the WHERE clause of + * its own SELECT. This rewriter runs first in rewrite, while subqueries, views and CTEs still have + * their own plan nodes. It gives every SELECT that calls score() a new score id and marks with it + * the score() calls of the SELECT and the MATCH and SEARCH predicates of its filters. + * BindScoreToOlapScan later binds the calls to the scan that those predicates filter. + * + * <p>A MATCH or SEARCH without a score id only filters rows: it adds nothing to the score of its + * scan, even when a filter of an outer SELECT is pushed down onto that scan. */ -public class CheckScoreUsage implements RewriteRuleFactory { - private static final Logger LOG = LogManager.getLogger(CheckScoreUsage.class); +public class CheckScoreUsage implements CustomRewriter { + public static final String NO_SCORING_PREDICATE_ERROR = + "score() requires a MATCH or SEARCH predicate in the WHERE clause of the same SELECT"; + public static final String UNSUPPORTED_CLAUSE_ERROR = "score() can only be used in the SELECT list " + + "and in the WHERE, GROUP BY, HAVING, QUALIFY and ORDER BY clauses"; @Override - public List<Rule> buildRules() { - return ImmutableList.of( - logicalProject(any()) - .when(project -> { - boolean hasScore = hasScoreFunction(project); - boolean isOptimized = isScoreAlreadyOptimized(project); - LOG.debug("Project check - hasScore: " + hasScore + ", isOptimized: " + isOptimized); - return hasScore && !isOptimized; - }) - .then(project -> { - throw new AnalysisException( - "score() function requires WHERE clause with MATCH function, " - + "ORDER BY and LIMIT for optimization"); - }).toRule(RuleType.CHECK_SCORE_USAGE), - - logicalAggregate(any()) - .when(this::hasScoreInAggregate) - .then(agg -> { - throw new AnalysisException( - "score() function cannot be used in aggregate functions. " - + "score() requires WHERE clause with MATCH function, " - + "ORDER BY and LIMIT for optimization"); - }).toRule(RuleType.CHECK_SCORE_USAGE) - ); + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!containsScore(plan)) { + return plan; + } + return tagSelect(plan); + } + + /** Whether any plan node of the tree calls score() in its own expressions. */ + public static boolean containsScore(Plan plan) { + return plan.anyMatch(node -> ((Plan) node).getExpressions().stream() + .anyMatch(expression -> expression.containsType(Score.class))); } - private boolean hasScoreFunction(LogicalProject<?> project) { - return project.getProjects().stream() - .anyMatch(projection -> { - if (projection instanceof Alias) { - return containsScoreFunction(((Alias) projection).child()); - } - return false; - }); + // A SELECT consists of the plan nodes from its root down to the roots of its subqueries, views, + // CTE producers and set operation inputs, which are SELECTs of their own. + private static boolean startsSelect(Plan parent, int childIndex) { + return parent instanceof LogicalSubQueryAlias || parent instanceof LogicalView + || parent instanceof LogicalCTEProducer || parent instanceof LogicalSetOperation + || parent instanceof LogicalRecursiveUnionAnchor || parent instanceof LogicalRecursiveUnionProducer + || parent instanceof LogicalApply && childIndex == 1; } - private boolean hasScoreInAggregate(LogicalAggregate<?> agg) { - boolean hasScoreInOutput = agg.getOutputExpressions().stream() - .anyMatch(this::containsScoreFunction); + private Plan tagSelect(Plan root) { + SelectSummary select = new SelectSummary(); + summarize(root, select); + if (!select.callsScore) { + return tag(root, Optional.empty()); + } + if (!select.hasScoringPredicate) { + throw new AnalysisException(NO_SCORING_PREDICATE_ERROR); + } + if (select.unsupportedClause) { + throw new AnalysisException(UNSUPPORTED_CLAUSE_ERROR); + } + return tag(root, Optional.of(StatementScopeIdGenerator.newExprId())); + } - boolean hasScoreInGroupBy = agg.getGroupByExpressions().stream() - .anyMatch(this::containsScoreFunction); + private void summarize(Plan plan, SelectSummary select) { + if (plan.getExpressions().stream().anyMatch(expression -> expression.containsType(Score.class))) { + select.callsScore = true; + select.unsupportedClause |= !canCallScore(plan); + } + if (plan instanceof LogicalFilter && ((LogicalFilter<?>) plan).getConjuncts().stream() + .anyMatch(conjunct -> conjunct.containsType(Match.class, Search.class))) { + select.hasScoringPredicate = true; + } + for (int i = 0; i < plan.arity(); i++) { + if (!startsSelect(plan, i)) { + summarize(plan.child(i), select); + } + } + } - return hasScoreInOutput || hasScoreInGroupBy; + // HAVING and QUALIFY are filters by now. Join conditions are rejected by CheckAnalysis. + private static boolean canCallScore(Plan plan) { + return plan instanceof LogicalProject || plan instanceof LogicalFilter + || plan instanceof LogicalAggregate || plan instanceof LogicalSort; } - private boolean containsScoreFunction(Expression expr) { - if (expr instanceof Score) { - return true; + private Plan tag(Plan plan, Optional<ExprId> scoreId) { + ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(plan.arity()); + boolean changed = false; + for (int i = 0; i < plan.arity(); i++) { + Plan child = plan.child(i); + Plan newChild = startsSelect(plan, i) ? tagSelect(child) : tag(child, scoreId); + changed |= newChild != child; + newChildren.add(newChild); } - return expr.children().stream().anyMatch(this::containsScoreFunction); + Plan newPlan = changed ? plan.withChildren(newChildren.build()) : plan; + return scoreId.isPresent() ? tagExpressions(newPlan, scoreId.get()) : newPlan; } - private boolean isScoreAlreadyOptimized(LogicalProject<?> project) { - Plan child = project.child(); - if (child instanceof LogicalOlapScan) { - LogicalOlapScan scan = (LogicalOlapScan) child; - return scan.getVirtualColumns().stream() - .anyMatch(virtualCol -> { - if (virtualCol instanceof Alias) { - Expression childExpr = ((Alias) virtualCol).child(); - return containsScoreFunction(childExpr); - } - return false; - }); + private Plan tagExpressions(Plan plan, ExprId scoreId) { + if (plan.getExpressions().stream() + .noneMatch(expression -> expression.containsType(Score.class, Match.class, Search.class))) { + return plan; } - return false; + if (plan instanceof LogicalFilter) { + LogicalFilter<?> filter = (LogicalFilter<?>) plan; + return filter.withConjuncts(filter.getConjuncts().stream() + .map(conjunct -> tagScoreAndPredicates(conjunct, scoreId)) Review Comment: The QUALIFY alias case from the latest review is fixed in 8fc7d7cf35. HAVING and QUALIFY now also cannot read score(), MATCH or SEARCH through an alias of the select list or GROUP BY, or through an alias of such an alias. Your `QUALIFY m AND rn = 1` query with `c1 MATCH 'red' AS m` now fails with the HAVING/QUALIFY error in `test_score_filter_usage` and `BindScoreToOlapScanTest`. The unit test also covers `HAVING m` and `GROUP BY m HAVING m`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
