XieJiann commented on code in PR #13902:
URL: https://github.com/apache/doris/pull/13902#discussion_r1013581490
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/joinreorder/hypergraph/Edge.java:
##########
@@ -22,20 +22,19 @@
import java.util.BitSet;
class Edge {
- final int index;
+ int index;
Review Comment:
done
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/joinreorder/hypergraph/GraphSimplifier.java:
##########
@@ -0,0 +1,383 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.joinreorder.hypergraph;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.memo.Group;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.stats.StatsCalculator;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+
+import com.google.common.base.Preconditions;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.PriorityQueue;
+
+/**
+ * GraphSimplifier is used to simplify HyperGraph {@link HyperGraph}
+ */
+public class GraphSimplifier {
+ class SimplificationStep {
+ double benefit;
+ int beforeIndex;
+ int afterIndex;
+ BitSet newLeft;
+ BitSet newRight;
+ Plan leftPlan;
+ Plan rightPlan;
+
+ SimplificationStep(double benefit, int beforeIndex, int afterIndex,
Plan leftPlan, Plan rightPlan,
+ BitSet newLeft, BitSet newRight) {
+ this.afterIndex = afterIndex;
+ this.beforeIndex = beforeIndex;
+ this.benefit = benefit;
+ this.leftPlan = leftPlan;
+ this.rightPlan = rightPlan;
+ this.newLeft = newLeft;
+ this.newRight = newRight;
+ }
+
+ public double getBenefit() {
+ return benefit;
+ }
+ }
+
+ class BestSimplification implements Comparable<BestSimplification> {
+ int bestNeighbor = -1;
+ Optional<SimplificationStep> step = Optional.empty();
+ // This data whether to be added to the queue
+ boolean isInQueue = false;
+
+ @Override
+ public int compareTo(GraphSimplifier.BestSimplification o) {
+ Preconditions.checkArgument(step.isPresent());
+ return Double.compare(getBenefit(), o.getBenefit());
+ }
+
+ public double getBenefit() {
+ return step.get().getBenefit();
+ }
+
+ public void setStep(SimplificationStep step) {
+ this.step = Optional.of(step);
+ }
+
+ public SimplificationStep getStep() {
+ return step.get();
+ }
+ }
+
+ // The graph we are simplifying
+ HyperGraph graph;
+
+ // It stores the children plan of one edge.
+ // we don't store it in hyper graph, because it's just used for simulating
join.
+ // In fact, the graph simplifier just generate the partial order of join
operator.
+ List<Pair<Plan, Plan>> joinPlans = new ArrayList<>();
+
+ // This is used for cache the intermediate results when calculate the
benefit
+ // Note that we store it for the after. Because if we put B after A (t1
Join_A t2 Join_B t3),
+ // B is changed. Therefore, Any step that involves B need to be
recalculated.
+ List<BestSimplification> simplifications = new ArrayList<>();
+ PriorityQueue<BestSimplification> priorityQueue = new PriorityQueue<>();
+ int edgeSize;
+
+ GraphSimplifier(HyperGraph graph) {
+ this.graph = graph;
+ edgeSize = graph.edges.size();
+ for (int i = 0; i < edgeSize; i++) {
+ BestSimplification bestSimplification = new BestSimplification();
+ Plan leftPlan = graph.getPlan(graph.getEdge(i).getLeft());
+ Plan rightPlan = graph.getPlan(graph.getEdge(i).getRight());
+ joinPlans.add(Pair.of(leftPlan, rightPlan));
+ simplifications.add(bestSimplification);
+ }
Review Comment:
done
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/joinreorder/hypergraph/GraphSimplifier.java:
##########
@@ -0,0 +1,383 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.joinreorder.hypergraph;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.memo.Group;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.stats.StatsCalculator;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+
+import com.google.common.base.Preconditions;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.PriorityQueue;
+
+/**
+ * GraphSimplifier is used to simplify HyperGraph {@link HyperGraph}
+ */
+public class GraphSimplifier {
+ class SimplificationStep {
+ double benefit;
+ int beforeIndex;
+ int afterIndex;
+ BitSet newLeft;
+ BitSet newRight;
+ Plan leftPlan;
+ Plan rightPlan;
+
+ SimplificationStep(double benefit, int beforeIndex, int afterIndex,
Plan leftPlan, Plan rightPlan,
+ BitSet newLeft, BitSet newRight) {
+ this.afterIndex = afterIndex;
+ this.beforeIndex = beforeIndex;
+ this.benefit = benefit;
+ this.leftPlan = leftPlan;
+ this.rightPlan = rightPlan;
+ this.newLeft = newLeft;
+ this.newRight = newRight;
+ }
+
+ public double getBenefit() {
+ return benefit;
+ }
+ }
+
+ class BestSimplification implements Comparable<BestSimplification> {
+ int bestNeighbor = -1;
+ Optional<SimplificationStep> step = Optional.empty();
+ // This data whether to be added to the queue
+ boolean isInQueue = false;
+
+ @Override
+ public int compareTo(GraphSimplifier.BestSimplification o) {
+ Preconditions.checkArgument(step.isPresent());
+ return Double.compare(getBenefit(), o.getBenefit());
+ }
+
+ public double getBenefit() {
+ return step.get().getBenefit();
+ }
+
+ public void setStep(SimplificationStep step) {
+ this.step = Optional.of(step);
+ }
+
+ public SimplificationStep getStep() {
+ return step.get();
+ }
+ }
+
+ // The graph we are simplifying
+ HyperGraph graph;
+
+ // It stores the children plan of one edge.
+ // we don't store it in hyper graph, because it's just used for simulating
join.
+ // In fact, the graph simplifier just generate the partial order of join
operator.
+ List<Pair<Plan, Plan>> joinPlans = new ArrayList<>();
+
+ // This is used for cache the intermediate results when calculate the
benefit
+ // Note that we store it for the after. Because if we put B after A (t1
Join_A t2 Join_B t3),
+ // B is changed. Therefore, Any step that involves B need to be
recalculated.
+ List<BestSimplification> simplifications = new ArrayList<>();
+ PriorityQueue<BestSimplification> priorityQueue = new PriorityQueue<>();
+ int edgeSize;
+
+ GraphSimplifier(HyperGraph graph) {
+ this.graph = graph;
+ edgeSize = graph.edges.size();
+ for (int i = 0; i < edgeSize; i++) {
+ BestSimplification bestSimplification = new BestSimplification();
+ Plan leftPlan = graph.getPlan(graph.getEdge(i).getLeft());
+ Plan rightPlan = graph.getPlan(graph.getEdge(i).getRight());
+ joinPlans.add(Pair.of(leftPlan, rightPlan));
+ simplifications.add(bestSimplification);
+ }
+ }
+
+ /**
+ * This function init the first simplification step
+ */
+ public void initFirstStep() {
+ for (int i = 0; i < edgeSize; i += 2) {
+ processNeighbors(i, i + 1, edgeSize);
+ }
+ }
+
+ /**
+ * This function will repeatedly apply simplification steps util the
number of csg-cmp pair
+ * is under the limit.
+ * In hyper graph, counting the csg-cmp pair is more expensive than the
apply simplification step, therefore
+ * we use binary search to find the least step we should apply.
+ *
+ * @param limit the limit number of the csg-cmp pair
+ */
+ public boolean simplifyGraph(int limit) {
+ // Now we only support simplify graph until the hyperGraph only
contains one plan
+ Preconditions.checkArgument(limit == 1);
+ while (applySimplificationStep()) {}
+ return true;
+ }
+
+ /**
+ * This function apply a simplification step
+ *
+ * @return If there is no other steps, return false.
+ */
+ public boolean applySimplificationStep() {
+ if (priorityQueue.isEmpty()) {
+ return false;
+ }
+ BestSimplification bestSimplification = priorityQueue.poll();
+ SimplificationStep bestStep = bestSimplification.step.get();
+ // TODO: add circle detector to refuse the edge that can cause a circle
+ graph.modifyEdge(bestStep.afterIndex, bestStep.newLeft,
bestStep.newRight);
+ proposeJoin(bestStep.afterIndex, bestStep.leftPlan,
bestStep.rightPlan);
+ processNeighbors(bestStep.afterIndex, 0, edgeSize);
+ return true;
+ }
+
+ /**
+ * Process all neighbors and try to make simplification step
+ * Note that when a given ordering is less advantageous and dropped out,
+ * we need to call this function recursively to update all relative steps.
+ *
+ * @param edgeIndex1 The index of join operator that we need to process
its neighbors
+ * @param beginIndex The beginning index of the processing join operators
+ * @param endIndex The end index of the processing join operators
+ */
+ private void processNeighbors(int edgeIndex1, int beginIndex, int
endIndex) {
+ // Go through the neighbors with lower index, where the best
simplification is stored
+ // Because the best simplification is stored in the edge with lower
index, we need to update it recursively
+ // when we can't reset that best simplification
+ for (int edgeIndex2 = beginIndex; edgeIndex2 < Integer.max(endIndex,
edgeIndex1); edgeIndex2 += 2) {
+ BestSimplification bestSimplification =
simplifications.get(edgeIndex2);
+ Optional<SimplificationStep> optionalStep =
makeSimplificationStep(edgeIndex1, edgeIndex2);
+ if (!optionalStep.isPresent()) {
+ continue;
+ }
+ if (!proposeSimplificationStep(optionalStep.get(),
bestSimplification, edgeIndex1)) {
+ // Because edgeIndex2 < edgeIndex1, so the recursion is
guaranteed to terminate
+ processNeighbors(edgeIndex2, 0, edgeSize);
+ }
+ }
+
+ // Go through the neighbors with higher index, we only need to
recalculate all related steps
+ for (int edgeIndex2 = Integer.max(beginIndex, edgeIndex1 + 1);
edgeIndex2 < endIndex; edgeIndex2 += 2) {
+ BestSimplification bestSimplification =
simplifications.get(edgeIndex1);
+ bestSimplification.bestNeighbor = -1;
+ Optional<SimplificationStep> optionalStep =
makeSimplificationStep(edgeIndex1, edgeIndex2);
+ if (!optionalStep.isPresent()) {
+ continue;
+ }
+ proposeSimplificationStep(optionalStep.get(), bestSimplification,
edgeIndex2);
+ }
+ }
+
+ private boolean proposeSimplificationStep(SimplificationStep step,
BestSimplification bestSimplification,
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
