This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1d984e0ebb1 return residual expr of join (#28760)
1d984e0ebb1 is described below
commit 1d984e0ebb1010f58b860c3089254053566d5b13
Author: 谢健 <jianx...@gmail.com>
AuthorDate: Mon Dec 25 12:53:14 2023 +0800
return residual expr of join (#28760)
---
.../jobs/joinorder/hypergraph/HyperGraph.java | 158 +++++++++++++--------
.../jobs/joinorder/hypergraph/edge/Edge.java | 22 +++
.../jobs/joinorder/hypergraph/edge/FilterEdge.java | 15 --
.../mv/AbstractMaterializedViewRule.java | 6 +-
.../rules/exploration/mv/ComparisonResult.java | 94 ++++++++++++
.../nereids/rules/exploration/mv/StructInfo.java | 2 +-
.../joinorder/hypergraph/CompareOuterJoinTest.java | 13 +-
...uterJoinTest.java => PullupExpressionTest.java} | 108 ++++----------
.../rules/exploration/mv/BuildStructInfoTest.java | 4 +-
9 files changed, 257 insertions(+), 165 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
index e534fc1fa84..7bd33c64b3c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
@@ -27,6 +27,7 @@ import
org.apache.doris.nereids.jobs.joinorder.hypergraph.node.DPhyperNode;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -44,18 +45,21 @@ import org.apache.doris.nereids.util.PlanUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
+import java.util.Map.Entry;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
/**
* The graph is a join graph, whose node is the leaf plan and edge is a join
operator.
@@ -268,11 +272,11 @@ public class HyperGraph {
filterEdges.forEach(e -> {
if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes)
&&
!PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType()))
{
- e.addRejectJoin(joinEdge);
+ e.addRejectEdge(joinEdge);
}
if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes)
&&
!PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType()))
{
- e.addRejectJoin(joinEdge);
+ e.addRejectEdge(joinEdge);
}
});
}
@@ -289,9 +293,11 @@ public class HyperGraph {
JoinEdge childA = joinEdges.get(i);
if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) {
leftRequired = LongBitmap.newBitmapUnion(leftRequired,
childA.getLeftSubNodes(joinEdges));
+ childA.addRejectEdge(edgeB);
}
if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType()))
{
leftRequired = LongBitmap.newBitmapUnion(leftRequired,
childA.getRightSubNodes(joinEdges));
+ childA.addRejectEdge(edgeB);
}
}
@@ -299,9 +305,11 @@ public class HyperGraph {
JoinEdge childA = joinEdges.get(i);
if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) {
rightRequired = LongBitmap.newBitmapUnion(rightRequired,
childA.getRightSubNodes(joinEdges));
+ childA.addRejectEdge(edgeB);
}
if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType()))
{
rightRequired = LongBitmap.newBitmapUnion(rightRequired,
childA.getLeftSubNodes(joinEdges));
+ childA.addRejectEdge(edgeB);
}
}
edgeB.setLeftExtendedNodes(leftRequired);
@@ -593,57 +601,75 @@ public class HyperGraph {
* compare hypergraph
*
* @param viewHG the compared hyper graph
- * @return null represents not compatible, or return some expression which
can
- * be pull up from this hyper graph
+ * @return Comparison result
*/
- public @Nullable List<Expression> isLogicCompatible(HyperGraph viewHG,
LogicalCompatibilityContext ctx) {
- Map<Edge, Edge> queryToView = constructEdgeMap(viewHG,
ctx.getQueryToViewEdgeExpressionMapping());
+ public ComparisonResult isLogicCompatible(HyperGraph viewHG,
LogicalCompatibilityContext ctx) {
+ // 1 try to construct a map which can be mapped from edge to edge
+ Map<Edge, Edge> queryToView = constructMapWithNode(viewHG,
ctx.getQueryToViewNodeIDMapping());
- // All edge in view must have a mapped edge in query
- if (queryToView.size() != viewHG.edgeSize()) {
- return null;
+ // 2. compare them by expression and extract residual expr
+ ComparisonResult.Builder builder = new ComparisonResult.Builder();
+ ComparisonResult edgeCompareRes = compareEdgesWithExpr(queryToView,
ctx.getQueryToViewEdgeExpressionMapping());
+ if (edgeCompareRes.isInvalid()) {
+ return ComparisonResult.INVALID;
}
+ builder.addComparisonResult(edgeCompareRes);
- boolean allMatch = queryToView.entrySet().stream()
- .allMatch(entry ->
- compareEdgeWithNode(entry.getKey(), entry.getValue(),
ctx.getQueryToViewNodeIDMapping()));
- if (!allMatch) {
- return null;
+ // 3. pull join edge of view is no sense, so reject them
+ if (!queryToView.values().containsAll(viewHG.joinEdges)) {
+ return ComparisonResult.INVALID;
}
- // join edges must be identical
- boolean isJoinIdentical = joinEdges.stream()
- .allMatch(queryToView::containsKey);
- if (!isJoinIdentical) {
- return null;
+ // 4. process residual edges
+ List<Expression> residualQueryJoin =
+ processOrphanEdges(Sets.difference(Sets.newHashSet(joinEdges),
queryToView.keySet()));
+ if (residualQueryJoin == null) {
+ return ComparisonResult.INVALID;
}
+ builder.addQueryExpressions(residualQueryJoin);
- // extract all top filters
- List<FilterEdge> residualFilterEdges = filterEdges.stream()
- .filter(e -> !queryToView.containsKey(e))
- .collect(ImmutableList.toImmutableList());
- if (residualFilterEdges.stream().anyMatch(e -> !e.isTopFilter())) {
- return null;
+ List<Expression> residualQueryFilter =
+
processOrphanEdges(Sets.difference(Sets.newHashSet(filterEdges),
queryToView.keySet()));
+ if (residualQueryFilter == null) {
+ return ComparisonResult.INVALID;
}
- return residualFilterEdges.stream()
- .flatMap(e -> e.getExpressions().stream())
- .collect(ImmutableList.toImmutableList());
+ builder.addQueryExpressions(residualQueryFilter);
+
+ List<Expression> residualViewFilter =
+ processOrphanEdges(
+ Sets.difference(Sets.newHashSet(viewHG.filterEdges),
Sets.newHashSet(queryToView.values())));
+ if (residualViewFilter == null) {
+ return ComparisonResult.INVALID;
+ }
+ builder.addViewExpressions(residualViewFilter);
+
+ return builder.build();
}
- private Map<Edge, Edge> constructEdgeMap(HyperGraph viewHG,
Map<Expression, Expression> exprMap) {
- Map<Expression, Edge> exprToEdge = constructExprMap(viewHG);
- Map<Edge, Edge> queryToView = new HashMap<>();
- joinEdges.stream()
- .filter(e -> !e.getExpressions().isEmpty()
- && exprMap.containsKey(e.getExpression(0))
- && compareEdgeWithExpr(e,
exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap))
- .forEach(e -> queryToView.put(e,
exprToEdge.get(exprMap.get(e.getExpression(0)))));
- filterEdges.stream()
- .filter(e -> !e.getExpressions().isEmpty()
- && exprMap.containsKey(e.getExpression(0))
- && compareEdgeWithExpr(e,
exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap))
- .forEach(e -> queryToView.put(e,
exprToEdge.get(exprMap.get(e.getExpression(0)))));
- return queryToView;
+ private List<Expression> processOrphanEdges(Set<Edge> edges) {
+ List<Expression> expressions = new ArrayList<>();
+ for (Edge edge : edges) {
+ if (!edge.canPullUp()) {
+ return null;
+ }
+ expressions.addAll(edge.getExpressions());
+ }
+ return expressions;
+ }
+
+ private Map<Edge, Edge> constructMapWithNode(HyperGraph viewHG,
Map<Integer, Integer> nodeMap) {
+ // TODO use hash map to reduce loop
+ Map<Edge, Edge> joinEdgeMap = joinEdges.stream().map(qe -> {
+ Optional<JoinEdge> viewEdge = viewHG.joinEdges.stream()
+ .filter(ve -> compareEdgeWithNode(qe, ve,
nodeMap)).findFirst();
+ return Pair.of(qe, viewEdge);
+ }).filter(e ->
e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second.get()));
+ Map<Edge, Edge> filterEdgeMap = filterEdges.stream().map(qe -> {
+ Optional<FilterEdge> viewEdge = viewHG.filterEdges.stream()
+ .filter(ve -> compareEdgeWithNode(qe, ve,
nodeMap)).findFirst();
+ return Pair.of(qe, viewEdge);
+ }).filter(e ->
e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p ->
p.second.get()));
+ return ImmutableMap.<Edge,
Edge>builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build();
}
private boolean compareEdgeWithNode(Edge t, Edge o, Map<Integer, Integer>
nodeMap) {
@@ -686,24 +712,40 @@ public class HyperGraph {
return bitmap2 == newBitmap1;
}
- private boolean compareEdgeWithExpr(Edge t, Edge o, Map<Expression,
Expression> expressionMap) {
- if (t.getExpressions().size() != o.getExpressions().size()) {
- return false;
- }
- int size = t.getExpressions().size();
- for (int i = 0; i < size; i++) {
- if (!Objects.equals(expressionMap.get(t.getExpression(i)),
o.getExpression(i))) {
- return false;
+ private ComparisonResult compareEdgesWithExpr(Map<Edge, Edge>
queryToViewedgeMap,
+ Map<Expression, Expression> queryToView) {
+ ComparisonResult.Builder builder = new ComparisonResult.Builder();
+ for (Entry<Edge, Edge> e : queryToViewedgeMap.entrySet()) {
+ ComparisonResult res = compareEdgeWithExpr(e.getKey(),
e.getValue(), queryToView);
+ if (res.isInvalid()) {
+ return ComparisonResult.INVALID;
}
+ builder.addComparisonResult(res);
}
- return true;
+ return builder.build();
}
- private Map<Expression, Edge> constructExprMap(HyperGraph hyperGraph) {
- Map<Expression, Edge> exprToEdge = new HashMap<>();
- hyperGraph.joinEdges.forEach(edge ->
edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge)));
- hyperGraph.filterEdges.forEach(edge ->
edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge)));
- return exprToEdge;
+ private ComparisonResult compareEdgeWithExpr(Edge query, Edge view,
Map<Expression, Expression> queryToView) {
+ Set<? extends Expression> queryExprSet = query.getExpressionSet();
+ Set<? extends Expression> viewExprSet = view.getExpressionSet();
+
+ Set<Expression> equalViewExpr = new HashSet<>();
+ List<Expression> residualQueryExpr = new ArrayList<>();
+ for (Expression queryExpr : queryExprSet) {
+ if (queryToView.containsKey(queryExpr) &&
viewExprSet.contains(queryToView.get(queryExpr))) {
+ equalViewExpr.add(queryToView.get(queryExpr));
+ } else {
+ residualQueryExpr.add(queryExpr);
+ }
+ }
+ List<Expression> residualViewExpr =
ImmutableList.copyOf(Sets.difference(viewExprSet, equalViewExpr));
+ if (!residualViewExpr.isEmpty() && !view.canPullUp()) {
+ return ComparisonResult.INVALID;
+ }
+ if (!residualQueryExpr.isEmpty() && !query.canPullUp()) {
+ return ComparisonResult.INVALID;
+ }
+ return new ComparisonResult(residualQueryExpr, residualViewExpr);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
index 4b88ce9f60d..c47300922b5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
@@ -22,6 +22,8 @@ import
org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import com.google.common.collect.ImmutableSet;
+
import java.util.BitSet;
import java.util.List;
import java.util.Set;
@@ -51,6 +53,8 @@ public abstract class Edge {
// record all sub nodes behind in this operator. It's T function in paper
private final long subTreeNodes;
+ private long rejectNodes = 0;
+
/**
* Create simple edge.
*/
@@ -71,6 +75,10 @@ public abstract class Edge {
return LongBitmap.getCardinality(leftExtendedNodes) == 1 &&
LongBitmap.getCardinality(rightExtendedNodes) == 1;
}
+ public void addRejectEdge(Edge edge) {
+ rejectNodes = LongBitmap.newBitmapUnion(edge.getReferenceNodes(),
rejectNodes);
+ }
+
public void addLeftExtendNode(long left) {
this.leftExtendedNodes = LongBitmap.or(this.leftExtendedNodes, left);
}
@@ -171,6 +179,20 @@ public abstract class Edge {
public abstract List<? extends Expression> getExpressions();
+ public Set<? extends Expression> getExpressionSet() {
+ return ImmutableSet.copyOf(getExpressions());
+ }
+
+ public boolean canPullUp() {
+ // Only inner join and filter with none rejectNodes can be pull up
+ return rejectNodes == 0
+ && !(this instanceof JoinEdge && !((JoinEdge)
this).getJoinType().isInnerJoin());
+ }
+
+ public long getRejectNodes() {
+ return rejectNodes;
+ }
+
public Expression getExpression(int i) {
return getExpressions().get(i);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
index ec037871025..57c6d9660d0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.Set;
@@ -32,25 +31,11 @@ import java.util.Set;
*/
public class FilterEdge extends Edge {
private final LogicalFilter<? extends Plan> filter;
- private final List<Integer> rejectEdges;
public FilterEdge(LogicalFilter<? extends Plan> filter, int index,
BitSet childEdges, long subTreeNodes, long childRequireNodes) {
super(index, childEdges, new BitSet(), subTreeNodes,
childRequireNodes, 0L);
this.filter = filter;
- rejectEdges = new ArrayList<>();
- }
-
- public void addRejectJoin(JoinEdge joinEdge) {
- rejectEdges.add(joinEdge.getIndex());
- }
-
- public List<Integer> getRejectEdges() {
- return rejectEdges;
- }
-
- public boolean isTopFilter() {
- return rejectEdges.isEmpty();
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 90ebe567c46..c63e2d85af3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -136,12 +136,14 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(queryToViewTableMapping,
queryToViewSlotMapping,
queryStructInfo, viewStructInfo);
- List<Expression> pulledUpExpressions =
StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
+ ComparisonResult comparisonResult =
StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
compatibilityContext);
- if (pulledUpExpressions == null) {
+ if (comparisonResult.isInvalid()) {
logger.debug(currentClassName + " graph logical is not
equals so continue");
continue;
}
+ // TODO: Use set of list? And consider view expr
+ List<Expression> pulledUpExpressions =
ImmutableList.copyOf(comparisonResult.getQueryExpressions());
// set pulled up expression to queryStructInfo predicates and
update related predicates
if (!pulledUpExpressions.isEmpty()) {
queryStructInfo.addPredicates(pulledUpExpressions);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
new file mode 100644
index 00000000000..cc8284f33e6
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * comparison result of view and query
+ */
+public class ComparisonResult {
+ public static final ComparisonResult INVALID = new
ComparisonResult(ImmutableList.of(), ImmutableList.of(), false);
+ public static final ComparisonResult EMPTY = new
ComparisonResult(ImmutableList.of(), ImmutableList.of(), true);
+ private final boolean valid;
+ private final List<Expression> viewExpressions;
+ private final List<Expression> queryExpressions;
+
+ public ComparisonResult(List<Expression> queryExpressions,
List<Expression> viewExpressions) {
+ this(queryExpressions, viewExpressions, true);
+ }
+
+ ComparisonResult(List<Expression> queryExpressions, List<Expression>
viewExpressions, boolean valid) {
+ this.viewExpressions = ImmutableList.copyOf(viewExpressions);
+ this.queryExpressions = ImmutableList.copyOf(queryExpressions);
+ this.valid = valid;
+ }
+
+ public List<Expression> getViewExpressions() {
+ return viewExpressions;
+ }
+
+ public List<Expression> getQueryExpressions() {
+ return queryExpressions;
+ }
+
+ public boolean isInvalid() {
+ return !valid;
+ }
+
+ /**
+ * Builder
+ */
+ public static class Builder {
+ ImmutableList.Builder<Expression> queryBuilder = new
ImmutableList.Builder<>();
+ ImmutableList.Builder<Expression> viewBuilder = new
ImmutableList.Builder<>();
+ boolean valid = true;
+
+ /**
+ * add comparisonResult
+ */
+ public Builder addComparisonResult(ComparisonResult comparisonResult) {
+ if (comparisonResult.isInvalid()) {
+ valid = false;
+ return this;
+ }
+ queryBuilder.addAll(comparisonResult.getQueryExpressions());
+ viewBuilder.addAll(comparisonResult.getViewExpressions());
+ return this;
+ }
+
+ public Builder addQueryExpressions(Collection<Expression> expressions)
{
+ queryBuilder.addAll(expressions);
+ return this;
+ }
+
+ public Builder addViewExpressions(Collection<Expression> expressions) {
+ viewBuilder.addAll(expressions);
+ return this;
+ }
+
+ public ComparisonResult build() {
+ return new ComparisonResult(queryBuilder.build(),
viewBuilder.build(), valid);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 3c9814cdce3..20da9ee12fd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -263,7 +263,7 @@ public class StructInfo {
* For inner join should judge only the join tables,
* for other join type should also judge the join direction, it's input
filter that can not be pulled up etc.
*/
- public static @Nullable List<Expression> isGraphLogicalEquals(StructInfo
queryStructInfo, StructInfo viewStructInfo,
+ public static ComparisonResult isGraphLogicalEquals(StructInfo
queryStructInfo, StructInfo viewStructInfo,
LogicalCompatibilityContext compatibilityContext) {
return
queryStructInfo.hyperGraph.isLogicCompatible(viewStructInfo.hyperGraph,
compatibilityContext);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index d0e5084a1c1..cfc88b2aa3c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -62,7 +62,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1,
p2)) != null);
+ Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1,
p2)).isInvalid());
}
@Test
@@ -79,7 +79,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1,
p2)) != null);
+ Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1,
p2)).isInvalid());
}
@Test
@@ -104,7 +104,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
+ List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2)).getQueryExpressions();
Assertions.assertEquals(1, exprList.size());
Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
}
@@ -132,7 +132,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
+ List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2)).getQueryExpressions();
Assertions.assertEquals(0, exprList.size());
}
@@ -159,7 +159,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
+ List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2)).getQueryExpressions();
Assertions.assertEquals(1, exprList.size());
Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
}
@@ -187,8 +187,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
- Assertions.assertEquals(null, exprList);
+ Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1,
p2)).isInvalid());
}
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
similarity index 54%
copy from
fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
copy to
fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
index d0e5084a1c1..d4818a844e8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -20,73 +20,23 @@ package org.apache.doris.nereids.jobs.joinorder.hypergraph;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.RuleSet;
import
org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
-import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.util.HyperGraphBuilder;
import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import java.util.List;
-
-class CompareOuterJoinTest extends SqlTestBase {
- @Test
- void testStarGraphWithInnerJoin() {
- // t2
- // |
- //t3-- t1 -- t4
- // |
- // t5
- CascadesContext c1 = createCascadesContext(
- "select * from T1, T2, T3, T4 where "
- + "T1.id = T2.id "
- + "and T1.id = T3.id "
- + "and T1.id = T4.id ",
- connectContext
- );
- Plan p1 = PlanChecker.from(c1)
- .analyze()
- .rewrite()
- .getPlan().child(0);
- Plan p2 = PlanChecker.from(c1)
- .analyze()
- .rewrite()
- .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
- .getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
- HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1,
p2)) != null);
- }
-
- @Test
- void testRandomQuery() {
- Plan p1 = new HyperGraphBuilder().randomBuildPlanWith(3, 3);
- p1 = PlanChecker.from(connectContext, p1)
- .analyze()
- .rewrite()
- .getPlan();
- Plan p2 = PlanChecker.from(connectContext, p1)
- .analyze()
- .rewrite()
- .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
- .getAllPlan().get(0);
- HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
- HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1,
p2)) != null);
- }
-
+class PullupExpressionTest extends SqlTestBase {
@Test
- void testInnerJoinWithFilter() {
-
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+ void testPullUpQueryFilter() {
CascadesContext c1 = createCascadesContext(
- "select * from T1 inner join T2 on T1.id = T2.id where T1.id =
0",
+ "select * from T1 join T2 on T1.id = T2.id where T1.id = 1",
connectContext
);
Plan p1 = PlanChecker.from(c1)
@@ -94,7 +44,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.getPlan().child(0);
CascadesContext c2 = createCascadesContext(
- "select * from T1 inner join T2 on T1.id = T2.id",
+ "select * from T1 join T2 on T1.id = T2.id",
connectContext
);
Plan p2 = PlanChecker.from(c2)
@@ -104,17 +54,16 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
- Assertions.assertEquals(1, exprList.size());
- Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
+ ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1,
p2));
+ Assertions.assertEquals(2, res.getQueryExpressions().size());
+ Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
+ Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(1).toSql());
}
- @Disabled
@Test
- void testInnerJoinWithFilter2() {
-
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+ void testPullUpQueryJoinCondition() {
CascadesContext c1 = createCascadesContext(
- "select * from T1 inner join T2 on T1.id = T2.id where T1.id =
0",
+ "select * from T1 join T2 on T1.id = T2.id and T1.score =
T2.score",
connectContext
);
Plan p1 = PlanChecker.from(c1)
@@ -122,7 +71,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.getPlan().child(0);
CascadesContext c2 = createCascadesContext(
- "select * from T1 inner join T2 on T1.id = T2.id where T1.id =
0",
+ "select * from T1 join T2 on T1.id = T2.id",
connectContext
);
Plan p2 = PlanChecker.from(c2)
@@ -132,24 +81,23 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
- Assertions.assertEquals(0, exprList.size());
+ ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1,
p2));
+ Assertions.assertEquals(1, res.getQueryExpressions().size());
+ Assertions.assertEquals("(score = score)",
res.getQueryExpressions().get(0).toSql());
}
@Test
- void testLeftOuterJoinWithLeftFilter() {
-
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+ void testPullUpViewFilter() {
CascadesContext c1 = createCascadesContext(
- "select * from ( select * from T1 where T1.id = 0) T1 left
outer join T2 on T1.id = T2.id",
+ "select * from T1 join T2 on T1.id = T2.id",
connectContext
);
-
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
Plan p1 = PlanChecker.from(c1)
.analyze()
.rewrite()
.getPlan().child(0);
CascadesContext c2 = createCascadesContext(
- "select * from T1 left outer join T2 on T1.id = T2.id",
+ "select * from T1 join T2 on T1.id = T2.id where T1.id = 1 and
T2.id = 1",
connectContext
);
Plan p2 = PlanChecker.from(c2)
@@ -159,25 +107,24 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
- Assertions.assertEquals(1, exprList.size());
- Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
+ ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1,
p2));
+ Assertions.assertEquals(2, res.getViewExpressions().size());
+ Assertions.assertEquals("(id = 1)",
res.getViewExpressions().get(0).toSql());
+ Assertions.assertEquals("(id = 1)",
res.getViewExpressions().get(1).toSql());
}
@Test
- void testLeftOuterJoinWithRightFilter() {
-
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+ void testPullUpViewJoinCondition() {
CascadesContext c1 = createCascadesContext(
- "select * from T1 left outer join ( select * from T2 where
T2.id = 0) T2 on T1.id = T2.id",
+ "select * from T1 join T2 on T1.id = T2.id ",
connectContext
);
-
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
Plan p1 = PlanChecker.from(c1)
.analyze()
.rewrite()
.getPlan().child(0);
CascadesContext c2 = createCascadesContext(
- "select * from T1 left outer join T2 on T1.id = T2.id",
+ "select * from T1 join T2 on T1.id = T2.id and T1.score =
T2.score",
connectContext
);
Plan p2 = PlanChecker.from(c2)
@@ -187,8 +134,9 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
- List<Expression> exprList = h1.isLogicCompatible(h2,
constructContext(p1, p2));
- Assertions.assertEquals(null, exprList);
+ ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1,
p2));
+ Assertions.assertEquals(1, res.getViewExpressions().size());
+ Assertions.assertEquals("(score = score)",
res.getViewExpressions().get(0).toSql());
}
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
index 6ac41a49812..f68365f6708 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
@@ -79,7 +79,7 @@ class BuildStructInfoTest extends SqlTestBase {
.when(j -> {
HyperGraph structInfo =
HyperGraph.toStructInfo(j).get(0);
Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
- Assertions.assertEquals(0, (int)
structInfo.getFilterEdge(0).getRejectEdges().get(0));
+ Assertions.assertEquals(3L,
structInfo.getFilterEdge(0).getRejectNodes());
return true;
}));
@@ -92,7 +92,7 @@ class BuildStructInfoTest extends SqlTestBase {
.when(j -> {
HyperGraph structInfo =
HyperGraph.toStructInfo(j).get(0);
Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
-
Assertions.assertTrue(structInfo.getFilterEdge(0).getRejectEdges().isEmpty());
+ Assertions.assertEquals(0,
structInfo.getFilterEdge(0).getRejectNodes());
return true;
}));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
