Repository: kylin Updated Branches: refs/heads/mhb22 [created] 1f1a7a869
temp Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f6fbd47 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f6fbd47 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f6fbd47 Branch: refs/heads/mhb22 Commit: 8f6fbd478430170090203c79ee4d256c838b31b8 Parents: 85313ee Author: Hongbin Ma <mahong...@apache.org> Authored: Thu Dec 22 09:59:43 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Dec 22 20:33:35 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 6 +++- .../gtrecord/StorageResponseGTScatter.java | 3 +- .../apache/kylin/rest/service/QueryService.java | 30 +++++--------------- 3 files changed, 14 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index a1c9050..ceda95c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true")); } - @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold + @Deprecated //Limit is good even it's large. This config is meaning less since we already have scan threshold public int getStoragePushDownLimitMax() { return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000")); } @@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000")); } + public int getLargeQueryThreshold() { + return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf((int) (getScanThreshold() * 0.1)))); + } + public int getDerivedInThreshold() { return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java index fe1afd3..88116b5 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java @@ -67,7 +67,8 @@ public class StorageResponseGTScatter implements IGTScanner { @Override public void close() throws IOException { - //do nothing + //If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource + Iterators. } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 8810c85..82bc407 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -19,27 +19,8 @@ package org.apache.kylin.rest.service; import java.io.IOException; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import java.sql.*; +import java.util.*; import javax.annotation.PostConstruct; import javax.sql.DataSource; @@ -360,8 +341,11 @@ public class QueryService extends BasicService { sqlResponse.setDuration(System.currentTimeMillis() - startTime); logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", // String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount())); - if (queryCacheEnabled && !sqlResponse.getIsException() // - && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) { + if (queryCacheEnabled && // + !sqlResponse.getIsException() && // + (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold) && // + (sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold())) //don't cache too large response + { cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse)); } } else {
