KYLIN-2377 Add kylin client query timeout
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a Branch: refs/heads/master-hbase0.98 Commit: 8263752a499158342e0588fda851a4006e8b1669 Parents: 0dcce15 Author: kangkaisen <kangkai...@live.com> Authored: Tue Jan 10 14:02:18 2017 +0800 Committer: kangkaisen <kangkai...@163.com> Committed: Wed Feb 8 15:51:30 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../java/org/apache/kylin/cube/CubeInstance.java | 1 + .../kylin/metadata/realization/IRealization.java | 5 ++++- .../org/apache/kylin/storage/StorageContext.java | 16 +++++++++++++++- .../storage/gtrecord/GTCubeStorageQueryBase.java | 2 ++ .../gtrecord/SequentialCubeTupleIterator.java | 5 +++++ .../apache/kylin/storage/hybrid/HybridInstance.java | 1 + 7 files changed, 32 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index fe15b1e..ebd9dfc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements Serializable { return udfMap; } + public int getQueryTimeoutSeconds() { + return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0")); + } + // ============================================================================ // SERVER // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 1d60575..fb9a7a7 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return sizeRecordSize; } + @Override public KylinConfig getConfig() { return config; } http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index a0243f4..aafc0f0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization; import java.util.List; import java.util.Set; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; @@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware { public DataModelDesc getModel(); public Set<TblColRef> getAllColumns(); - + public Set<ColumnDesc> getAllColumnDescs(); public List<TblColRef> getAllDimensions(); @@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware { public long getDateRangeEnd(); public boolean supportsLimitPushDown(); + + public KylinConfig getConfig(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index ec46f83..ab0ea73 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -41,6 +41,7 @@ public class StorageContext { private int finalPushDownLimit = Integer.MAX_VALUE; private boolean hasSort = false; private boolean acceptPartialResult = false; + private long deadline; private boolean exactAggregation = false; private boolean needStorageAggregation = false; @@ -123,6 +124,19 @@ public class StorageContext { } } + public long getDeadline() { + return this.deadline; + } + + public void setDeadline(IRealization realization) { + int timeout = realization.getConfig().getQueryTimeoutSeconds() * 1000; + if (timeout == 0) { + this.deadline = Long.MAX_VALUE; + } else { + this.deadline = timeout + System.currentTimeMillis(); + } + } + public void markSort() { this.hasSort = true; } @@ -202,5 +216,5 @@ public class StorageContext { public void setStorageQuery(IStorageQuery storageQuery) { this.storageQuery = storageQuery; } - + } http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 4fcfad1..4dbdf94 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -120,6 +120,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // set limit push down enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context); + // set query deadline + context.setDeadline(cubeInstance); // set cautious threshold to prevent out of memory setThresholdIfNecessary(dimensionsD, metrics, context); http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index c621215..c6b2c6c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -28,6 +28,7 @@ import javax.annotation.Nullable; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.gridtable.GTScanTimeoutException; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITuple; @@ -141,6 +142,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator { @Override public ITuple next() { + if (scanCount % 100 == 1 && System.currentTimeMillis() > context.getDeadline()) { + throw new GTScanTimeoutException("Query Timeout!"); + } + // prevent the big query to make the Query Server OOM if (scanCount++ > SCAN_THRESHOLD) { throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD); http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 4f0e446..1b113ee 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -251,6 +251,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization return getType() + "[name=" + name + "]"; } + @Override public KylinConfig getConfig() { return config; }
