add a toggle for dumpping storage partitions add more log
temp for dumpped partitions minor fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bfe8b1a3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bfe8b1a3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bfe8b1a3 Branch: refs/heads/master-cdh5.7 Commit: bfe8b1a35c6b2c2f377eeb72295622ecb4d71284 Parents: 596ab51 Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Dec 23 13:56:01 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Dec 27 22:17:33 2016 +0800 ---------------------------------------------------------------------- .../kylin/common/debug/BackdoorToggles.java | 30 ++++++++++++++++++++ .../gtrecord/StorageResponseGTScatter.java | 2 ++ 2 files changed, 32 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java index ca4a19c..8109cf2 100644 --- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java +++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java @@ -62,6 +62,14 @@ public class BackdoorToggles { return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR); } + public static String getPartitionDumpDir() { + return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR); + } + + public static String getDumpedPartitionDir() { + return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR); + } + public static int getQueryTimeout() { String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT); if (v == null) @@ -190,6 +198,28 @@ public class BackdoorToggles { */ public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT"; + /** + * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from storage. + * The dumped partitions are used for performance profiling, for example. + * + example:(put it into request body) + "backdoorToggles": { + "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping" + } + */ + public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = "DEBUG_TOGGLE_PARTITION_DUMP_DIR"; + + /** + * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to retrieve previously dumped partitions + * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR + * + example:(put it into request body) + "backdoorToggles": { + "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped" + } + */ + public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = "DEBUG_TOGGLE_DUMPED_PARTITION_DIR"; + // properties on statement may go with this "channel" too /** * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java index 6283340..dc8746f 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java @@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner { public Iterator<GTRecord> iterator() { Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks, new EndpointResponseGTScatterFunc()); if (storagePushDownLimit != Integer.MAX_VALUE) { + logger.info("Using SortedIteratorMergerWithLimit to merge partitions"); return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator(); } else { + logger.info("Using Iterators.concat to merge partitions"); return Iterators.concat(shardSubsets); } }
