This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5_beta
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f35edc39049d56bc2c8f36df922ca8e46462b7a9
Author: Pengfei Zhan <dethr...@gmail.com>
AuthorDate: Tue Apr 4 00:08:23 2023 +0800
KYLIN-5632 Move candidate sorting method to the QueryRouter
---
.../metadata/cube/cuboid/AggIndexMatcher.java | 9 ++--
.../org/apache/kylin/query/routing/Candidate.java | 62 +++++++--------------
.../kylin/query/routing/PartitionPruningRule.java | 2 +-
.../routing/{RoutingRule.java => PruningRule.java} | 7 ++-
.../apache/kylin/query/routing/QueryRouter.java | 63 +++++++++++++++++-----
.../kylin/query/routing/RealizationChooser.java | 11 +---
.../routing/RemoveIncapableRealizationsRule.java | 2 +-
.../kylin/query/routing/SegmentPruningRule.java | 2 +-
.../kylin/query/routing/CandidateSortTest.java | 11 +++-
9 files changed, 94 insertions(+), 75 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
index c2f0a0c194..5561581d72 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
@@ -29,6 +29,9 @@ import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.cube.model.IndexEntity;
@@ -45,10 +48,6 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -85,7 +84,7 @@ public class AggIndexMatcher extends IndexMatcher {
}
@Override
- MatchResult match(LayoutEntity layout) {
+ public MatchResult match(LayoutEntity layout) {
if (canSkipIndexMatch(layout.getIndex()) || !isValid()) {
return new MatchResult();
}
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
index 1f02e7df65..5fe45a9425 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -44,9 +44,6 @@ import lombok.Setter;
@Getter
public class Candidate {
- public static final CandidateComparator COMPARATOR = new
CandidateComparator();
- public static final CandidateTableIndexComparator COMPARATOR_TABLE_INDEX =
new CandidateTableIndexComparator();
-
//
============================================================================
IRealization realization;
@@ -102,55 +99,36 @@ public class Candidate {
return realization.toString();
}
- public static class CandidateComparator implements Comparator<Candidate> {
-
- @Override
- public int compare(Candidate c1, Candidate c2) {
- return compareCandidate(c1, c2);
- }
+ public static Comparator<Candidate> tableIndexUnmatchedColSizeSorter() {
+ return Comparator.comparingInt(c ->
c.getCapability().getLayoutUnmatchedColsSize());
}
- public static class CandidateTableIndexComparator implements
Comparator<Candidate> {
-
- @Override
- public int compare(Candidate c1, Candidate c2) {
- CapabilityResult capabilityResult1 = c1.getCapability();
- CapabilityResult capabilityResult2 = c2.getCapability();
- if (capabilityResult1.getLayoutUnmatchedColsSize() !=
capabilityResult2.getLayoutUnmatchedColsSize()) {
- return capabilityResult1.getLayoutUnmatchedColsSize() -
capabilityResult2.getLayoutUnmatchedColsSize();
+ public static Comparator<Candidate> modelPrioritySorter() {
+ return (c1, c2) -> {
+ if (QueryContext.current().getModelPriorities().length == 0) {
+ return 0;
}
- return compareCandidate(c1, c2);
- }
- }
-
- private static int compareCandidate(Candidate c1, Candidate c2) {
- IRealization real1 = c1.getRealization();
- IRealization real2 = c2.getRealization();
-
- if (QueryContext.current().getModelPriorities().length > 0) {
-
Map<String, Integer> priorities = new HashMap<>();
for (int i = 0; i <
QueryContext.current().getModelPriorities().length; i++) {
priorities.put(QueryContext.current().getModelPriorities()[i],
i);
}
- int comp =
priorities.getOrDefault(StringUtils.upperCase(real1.getModel().getAlias()),
Integer.MAX_VALUE)
- -
priorities.getOrDefault(StringUtils.upperCase(real2.getModel().getAlias()),
Integer.MAX_VALUE);
- if (comp != 0) {
- return comp;
- }
- }
+ String modelAlias1 =
StringUtils.upperCase(c1.getRealization().getModel().getAlias());
+ String modelAlias2 =
StringUtils.upperCase(c2.getRealization().getModel().getAlias());
+ return priorities.getOrDefault(modelAlias1, Integer.MAX_VALUE)
+ - priorities.getOrDefault(modelAlias2, Integer.MAX_VALUE);
+ };
+ }
- int comp = real1.getCost() - real2.getCost();
- if (comp != 0) {
- return comp;
- }
+ public static Comparator<Candidate> realizationCostSorter() {
+ return Comparator.comparingInt(c -> c.getRealization().getCost());
+ }
- comp = Double.compare(c1.capability.cost, c2.capability.cost);
- if (comp != 0) {
- return comp;
- }
+ public static Comparator<Candidate> realizationCapabilityCostSorter() {
+ return Comparator.comparingDouble(c -> c.getCapability().cost);
+ }
- return real1.getModel().getId().compareTo(real2.getModel().getId());
+ public static Comparator<Candidate> modelUuidSorter() {
+ return Comparator.comparing(c ->
c.getRealization().getModel().getId());
}
}
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/PartitionPruningRule.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/PartitionPruningRule.java
index c028906e47..9e0eb263c8 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/PartitionPruningRule.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/PartitionPruningRule.java
@@ -54,7 +54,7 @@ import org.apache.kylin.query.util.RexUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class PartitionPruningRule extends RoutingRule {
+public class PartitionPruningRule extends PruningRule {
private static final String NEED_PUSH_DOWN = "NULL";
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/PruningRule.java
similarity index 75%
rename from
src/query-common/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
rename to
src/query-common/src/main/java/org/apache/kylin/query/routing/PruningRule.java
index 0020b5c0a7..1719f517a4 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/PruningRule.java
@@ -19,12 +19,15 @@
package org.apache.kylin.query.routing;
/**
+ * The pruning rule is responsible for matching the indexes that can be used
for querying.
+ * Classes extend this class do not have any attributes and only needs to
implement
+ * the {@link PruningRule#apply(Candidate)} method.
*/
-public abstract class RoutingRule {
+public abstract class PruningRule {
@Override
public String toString() {
- return this.getClass().toString();
+ return this.getClass().getName();
}
public abstract void apply(Candidate candidate);
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index 9b3d4c03f7..1105ee4e9c 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -18,30 +18,69 @@
package org.apache.kylin.query.routing;
+import java.util.Comparator;
import java.util.List;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Ordering;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.query.relnode.OLAPContext;
+
+import lombok.Getter;
-/**
- * @author xjiang
- */
public class QueryRouter {
private QueryRouter() {
}
- private static final List<RoutingRule> LAYOUT_CHOOSING_RULES =
Lists.newLinkedList();
+ public static void applyRules(Candidate candidate) {
+ Strategy pruningStrategy = getStrategy(candidate.getCtx());
+ for (PruningRule r : pruningStrategy.getRules()) {
+ r.apply(candidate);
+ }
+ }
- static {
- LAYOUT_CHOOSING_RULES.add(new SegmentPruningRule());
- LAYOUT_CHOOSING_RULES.add(new PartitionPruningRule());
- LAYOUT_CHOOSING_RULES.add(new RemoveIncapableRealizationsRule());
+ public static void sortCandidates(OLAPContext context, List<Candidate>
candidates) {
+ Strategy strategy = getStrategy(context);
+ candidates.sort(strategy.getSorter());
}
- public static void applyRules(Candidate candidate) {
- for (RoutingRule rule : LAYOUT_CHOOSING_RULES) {
- rule.apply(candidate);
- }
+ private static Strategy getStrategy(OLAPContext context) {
+ String project = context.olapSchema.getProjectName();
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
+ return new Strategy(projectConfig);
}
+ public static class Strategy {
+ private static final PruningRule SEGMENT_PRUNING = new
SegmentPruningRule();
+ private static final PruningRule PARTITION_PRUNING = new
PartitionPruningRule();
+ private static final PruningRule REMOVE_INCAPABLE_REALIZATIONS = new
RemoveIncapableRealizationsRule();
+
+ @Getter
+ List<PruningRule> rules = Lists.newArrayList();
+
+ private final List<Comparator<Candidate>> sorters =
Lists.newArrayList();
+
+ public Comparator<Candidate> getSorter() {
+ return Ordering.compound(sorters);
+ }
+
+ public Strategy(KylinConfig config) {
+
+ // add all rules
+ rules.add(SEGMENT_PRUNING);
+ rules.add(PARTITION_PRUNING);
+ rules.add(REMOVE_INCAPABLE_REALIZATIONS);
+
+ // add all sorters
+ if (config.useTableIndexAnswerSelectStarEnabled()) {
+ sorters.add(Candidate.tableIndexUnmatchedColSizeSorter());
+ }
+ sorters.add(Candidate.modelPrioritySorter());
+ sorters.add(Candidate.realizationCostSorter());
+ sorters.add(Candidate.realizationCapabilityCostSorter());
+ sorters.add(Candidate.modelUuidSorter());
+ }
+ }
}
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index e6fe2878bd..13720f8932 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -227,7 +227,7 @@ public class RealizationChooser {
}
// Step 3. find the lowest-cost candidate
- sortCandidate(context, candidates);
+ QueryRouter.sortCandidates(context, candidates);
logger.trace("Cost Sorted Realizations {}", candidates);
Candidate candidate = candidates.get(0);
restoreOLAPContextProps(context, candidate.getRewrittenCtx());
@@ -286,15 +286,6 @@ public class RealizationChooser {
return candidates;
}
- private static void sortCandidate(OLAPContext context, List<Candidate>
candidates) {
- KylinConfig projectConfig =
NProjectManager.getProjectConfig(context.olapSchema.getProjectName());
- if (projectConfig.useTableIndexAnswerSelectStarEnabled() &&
context.getSQLDigest().isRawQuery) {
- candidates.sort(Candidate.COMPARATOR_TABLE_INDEX);
- } else {
- candidates.sort(Candidate.COMPARATOR);
- }
- }
-
private static void checkNoRealizationWithStreaming(OLAPContext context) {
String projectName = context.olapSchema.getProjectName();
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RemoveIncapableRealizationsRule.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/RemoveIncapableRealizationsRule.java
index 284f8e31b7..29886fb4aa 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RemoveIncapableRealizationsRule.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/RemoveIncapableRealizationsRule.java
@@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j;
/**
*/
@Slf4j
-public class RemoveIncapableRealizationsRule extends RoutingRule {
+public class RemoveIncapableRealizationsRule extends PruningRule {
@Override
public void apply(Candidate candidate) {
if (candidate.getCapability() != null) {
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/SegmentPruningRule.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/SegmentPruningRule.java
index 6c104655b1..3464271cc2 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/SegmentPruningRule.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/SegmentPruningRule.java
@@ -74,7 +74,7 @@ import lombok.var;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class SegmentPruningRule extends RoutingRule {
+public class SegmentPruningRule extends PruningRule {
private static final TimeZone UTC_ZONE = TimeZone.getTimeZone("UTC");
diff --git
a/src/query/src/test/java/org/apache/kylin/query/routing/CandidateSortTest.java
b/src/query/src/test/java/org/apache/kylin/query/routing/CandidateSortTest.java
index 993837a010..f9799f42ff 100644
---
a/src/query/src/test/java/org/apache/kylin/query/routing/CandidateSortTest.java
+++
b/src/query/src/test/java/org/apache/kylin/query/routing/CandidateSortTest.java
@@ -19,12 +19,15 @@
package org.apache.kylin.query.routing;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Ordering;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -135,8 +138,14 @@ public class CandidateSortTest {
}
private SortedCandidate sort(Candidate... candidates) {
+ List<Comparator<Candidate>> sorters = Lists.newArrayList();
+ sorters.add(Candidate.modelPrioritySorter());
+ sorters.add(Candidate.realizationCostSorter());
+ sorters.add(Candidate.realizationCapabilityCostSorter());
+ sorters.add(Candidate.modelUuidSorter());
+
return candidate -> {
- Arrays.sort(candidates, Candidate.COMPARATOR);
+ Arrays.sort(candidates, Ordering.compound(sorters));
Assert.assertEquals(candidate.getRealization().getModel().getAlias(),
candidates[0].getRealization().getModel().getAlias());
};
