Repository: kylin Updated Branches: refs/heads/2.0-rc 0ba1e065c -> 183b31a66
KYLIN-1226 A better wait GC done Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/183b31a6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/183b31a6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/183b31a6 Branch: refs/heads/2.0-rc Commit: 183b31a664e4a79ca889462a5d8e4d5d69d7dc5a Parents: 0ba1e06 Author: Li, Yang <yang...@ebay.com> Authored: Mon Dec 14 15:20:19 2015 +0800 Committer: Li, Yang <yang...@ebay.com> Committed: Mon Dec 14 15:20:49 2015 +0800 ---------------------------------------------------------------------- .../common/util/MemoryBudgetController.java | 24 +++++++++++++++++ .../cube/inmemcubing/InMemCubeBuilder.java | 28 +++++--------------- 2 files changed, 31 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/183b31a6/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java index e454414..d94f32b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java @@ -233,6 +233,30 @@ public class MemoryBudgetController { return true; } + public static int gcAndGetSystemAvailMB() { + final int tolerance = 1; + try { + int lastMB = -1; + while (true) { + Runtime.getRuntime().gc(); + Thread.sleep(1000); + int thisMB = getSystemAvailMB(); + + if (lastMB < 0) { + lastMB = thisMB; + continue; + } + if (lastMB - thisMB < tolerance) { + return thisMB; + } + lastMB = thisMB; + } + } catch (InterruptedException e) { + logger.error("", e); + return getSystemAvailMB(); + } + } + public static long getSystemAvailBytes() { Runtime runtime = Runtime.getRuntime(); long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process http://git-wip-us.apache.org/repos/asf/kylin/blob/183b31a6/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index a3e3a03..97c96fb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -295,21 +295,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { } private int getSystemAvailMB() { - // GC to be precise on memory left - Runtime.getRuntime().gc(); - try { - Thread.sleep(2500); - } catch (InterruptedException e) { - logger.error("", e); - } - // GC again to be precise on memory left - Runtime.getRuntime().gc(); - try { - Thread.sleep(2500); - } catch (InterruptedException e) { - logger.error("", e); - } - return MemoryBudgetController.getSystemAvailMB(); + return MemoryBudgetController.gcAndGetSystemAvailMB(); } private void makeMemoryBudget() { @@ -330,20 +316,20 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { } private CuboidResult createBaseCuboid(BlockingQueue<List<String>> input) throws IOException { + int mbBefore = getSystemAvailMB(); + int mbAfter = 0; + + long startTime = System.currentTimeMillis(); + logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB"); + GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); - int mbBefore = getSystemAvailMB(); - int mbAfter = 0; - Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true); - long startTime = System.currentTimeMillis(); - logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB"); - int count = 0; for (GTRecord r : aggregationScanner) { if (mbAfter == 0) {
