This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fbbe988f8ccc41166c427d16a39d7eaa4e2de36e
Author: minghong <engle...@gmail.com>
AuthorDate: Wed Dec 21 21:10:47 2022 +0800
[opt](planner) add session var: COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD
(#15225)
in previous pr(#14876) we compact equals like "a=1 or a=2 or a = 3 " in to
"a in (1, 2, 3)"
this pr set a lower bound for the number of equals
COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD (default is 2)
for performance reason, we create a hashSet to collect literals, like
{1,2,3}. and hence, the literals in "in-predicates" are in random order.
for regression test, if we need stable output of explain string, set
COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD to a large number to avoid compact rule.
---
.../src/main/java/org/apache/doris/qe/SessionVariable.java | 14 ++++++++++++++
.../doris/rewrite/CompactEqualsToInPredicateRule.java | 12 +++++++++---
.../data/performance_p0/redundant_conjuncts.out | 2 +-
.../suites/performance_p0/redundant_conjuncts.groovy | 1 +
4 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 344aaf1aba..b7f30dc7c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -196,6 +196,9 @@ public class SessionVariable implements Serializable,
Writable {
//percentage of EXEC_MEM_LIMIT
public static final String BROADCAST_HASHTABLE_MEM_LIMIT_PERCENTAGE =
"broadcast_hashtable_mem_limit_percentage";
+
+ public static final String COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD =
"compact_equal_to_in_predicate_threshold";
+
public static final String NEREIDS_STAR_SCHEMA_SUPPORT =
"nereids_star_schema_support";
public static final String NEREIDS_CBO_PENALTY_FACTOR =
"nereids_cbo_penalty_factor";
@@ -542,6 +545,9 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = NEREIDS_STAR_SCHEMA_SUPPORT)
private boolean nereidsStarSchemaSupport = true;
+ @VariableMgr.VarAttr(name = COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD)
+ private int compactEqualToInPredicateThreshold = 2;
+
@VariableMgr.VarAttr(name = NEREIDS_CBO_PENALTY_FACTOR)
private double nereidsCboPenaltyFactor = 0.7;
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TRACE)
@@ -656,6 +662,14 @@ public class SessionVariable implements Serializable,
Writable {
this.blockEncryptionMode = blockEncryptionMode;
}
+ public void setCompactEqualToInPredicateThreshold(int threshold) {
+ this.compactEqualToInPredicateThreshold = threshold;
+ }
+
+ public int getCompactEqualToInPredicateThreshold() {
+ return compactEqualToInPredicateThreshold;
+ }
+
public long getMaxExecMemByte() {
return maxExecMemByte;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
index 7375b83121..2b8ad5ed2c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
@@ -27,6 +27,7 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter.ClauseType;
import com.google.common.collect.Lists;
@@ -44,7 +45,6 @@ a = 1 or a = 2 or a = 3 or a in (4, 5, 6) => a in (1, 2, 3,
4, 5, 6)
*/
public class CompactEqualsToInPredicateRule implements ExprRewriteRule {
public static CompactEqualsToInPredicateRule INSTANCE = new
CompactEqualsToInPredicateRule();
- private static final int COMPACT_SIZE = 2;
@Override
public Expr apply(Expr expr, Analyzer analyzer, ClauseType clauseType)
throws AnalysisException {
@@ -67,9 +67,15 @@ public class CompactEqualsToInPredicateRule implements
ExprRewriteRule {
expr in form of A or B or ...
*/
private Pair<Boolean, Expr> compactEqualsToInPredicate(Expr expr) {
+ int compactThreshold;
+ if (ConnectContext.get() == null) {
+ compactThreshold = 2;
+ } else {
+ compactThreshold =
ConnectContext.get().getSessionVariable().getCompactEqualToInPredicateThreshold();
+ }
boolean changed = false;
List<Expr> disConjuncts = getDisconjuncts(expr);
- if (disConjuncts.size() < COMPACT_SIZE) {
+ if (disConjuncts.size() < compactThreshold) {
return Pair.of(false, expr);
}
Map<SlotRef, Set<Expr>> equalMap = new HashMap<>();
@@ -111,7 +117,7 @@ public class CompactEqualsToInPredicateRule implements
ExprRewriteRule {
for (Entry<SlotRef, Set<Expr>> entry : equalMap.entrySet()) {
SlotRef slot = entry.getKey();
InPredicate in = inPredMap.get(slot);
- if (entry.getValue().size() >= COMPACT_SIZE || in != null) {
+ if (entry.getValue().size() >= compactThreshold || in != null) {
if (in == null) {
in = new InPredicate(entry.getKey(),
Lists.newArrayList(entry.getValue()), false);
inPredMap.put(slot, in);
diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out
b/regression-test/data/performance_p0/redundant_conjuncts.out
index 9cba503956..98178f31aa 100644
--- a/regression-test/data/performance_p0/redundant_conjuncts.out
+++ b/regression-test/data/performance_p0/redundant_conjuncts.out
@@ -23,7 +23,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE:
default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts),
PREAGGREGATION: OFF. Reason: No AggregateInfo
- PREDICATES: `k1` IN (2, 1)
+ PREDICATES: (`k1` = 1 OR `k1` = 2)
partitions=0/1, tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
diff --git a/regression-test/suites/performance_p0/redundant_conjuncts.groovy
b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
index 14624a8049..3fbe2c9237 100644
--- a/regression-test/suites/performance_p0/redundant_conjuncts.groovy
+++ b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
@@ -39,6 +39,7 @@ suite("redundant_conjuncts") {
EXPLAIN SELECT v1 FROM redundant_conjuncts WHERE k1 = 1 AND k1 = 1;
"""
+ sql "set COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD = 100"
qt_redundant_conjuncts_gnerated_by_extract_common_filter """
EXPLAIN SELECT v1 FROM redundant_conjuncts WHERE k1 = 1 OR k1 = 2;
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
