Repository: kylin Updated Branches: refs/heads/master 2c2970feb -> e68b25796
KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e68b2579 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e68b2579 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e68b2579 Branch: refs/heads/master Commit: e68b257968a03809e114f6871394c4b55b05562f Parents: 2c2970f Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Nov 15 21:06:21 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Nov 15 21:06:25 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/KylinConfigBase.java | 5 ++++- .../storage/translate/DerivedFilterTranslator.java | 13 +++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e68b2579/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index ee9f57c..d44ff96 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -486,6 +486,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000")); } + public int getDerivedInThreshold() { + return Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20")); + } + public int getBadQueryStackTraceDepth() { return Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10")); } @@ -634,7 +638,6 @@ abstract public class KylinConfigBase implements Serializable { return this.getOptional("kylin.job.hive.database.for.intermediatetable", "default"); } - public boolean isHiveRedistributeEnabled() { return Boolean.parseBoolean(this.getOptional("kylin.job.hive.intermediatetable.redistribute.enabled", "true")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e68b2579/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java index 60c3a01..54b4c31 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java @@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate; import java.util.List; import java.util.Set; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.kv.RowKeyColumnOrder; @@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** * @author yangli9 - * */ public class DerivedFilterTranslator { - - private static final int IN_THRESHOLD = 5; + + private static final Logger logger = LoggerFactory.getLogger(DerivedFilterTranslator.class); public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) { @@ -83,7 +85,10 @@ public class DerivedFilterTranslator { TupleFilter translated; boolean loosened; - if (satisfyingHostRecords.size() > IN_THRESHOLD) { + if (satisfyingHostRecords.size() > KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) { + logger.info("Deciding to loosen filter on derived filter as host candidates number {} exceeds threshold {}",// + satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold() + ); translated = buildRangeFilter(hostCols, satisfyingHostRecords); loosened = true; } else {
