Repository: kylin Updated Branches: refs/heads/master ff8510313 -> 460536c00
KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/460536c0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/460536c0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/460536c0 Branch: refs/heads/master Commit: 460536c001575bd09e9f059062efe0d1049d6b31 Parents: ff85103 Author: gaodayue <gaoda...@meituan.com> Authored: Wed Sep 28 17:09:06 2016 +0800 Committer: gaodayue <gaoda...@meituan.com> Committed: Tue Oct 11 20:36:11 2016 +0800 ---------------------------------------------------------------------- core-cube/pom.xml | 5 + .../gridtable/AggregationCacheMemSizeTest.java | 290 +++++++++---------- core-metadata/pom.xml | 5 + .../measure/AggregatorMemEstimateTest.java | 104 +++++++ pom.xml | 35 ++- 5 files changed, 292 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/pom.xml ---------------------------------------------------------------------- diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 7c9a549..39bba59 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -69,6 +69,11 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.github.jbellis</groupId> + <artifactId>jamm</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java index fcd434e..00c0bd0 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java @@ -17,197 +17,195 @@ package org.apache.kylin.gridtable; -import java.math.BigDecimal; -import java.util.Comparator; -import java.util.Random; -import java.util.SortedMap; -import java.util.TreeMap; - +import com.google.common.base.Stopwatch; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.basic.BigDecimalSumAggregator; import org.apache.kylin.measure.basic.DoubleSumAggregator; import org.apache.kylin.measure.basic.LongSumAggregator; +import org.apache.kylin.measure.bitmap.BitmapAggregator; +import org.apache.kylin.measure.bitmap.BitmapCounter; import org.apache.kylin.measure.hllc.HLLCAggregator; import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; import org.apache.kylin.metadata.datatype.DoubleMutable; import org.apache.kylin.metadata.datatype.LongMutable; +import org.github.jamm.MemoryMeter; import org.junit.Test; +import java.math.BigDecimal; +import java.util.*; + public class AggregationCacheMemSizeTest { + private static final MemoryMeter meter = new MemoryMeter(); + private static final BitmapCounter[] bitmaps = new BitmapCounter[5]; + private static final Random random = new Random(); + + // consider bitmaps with variant cardinality + static { + for (int i = 0; i < bitmaps.length; i++) { + bitmaps[i] = new BitmapCounter(); + } - public static final int NUM_OF_OBJS = 1000000 / 2; + final int totalBits = 1_000_000; - interface CreateAnObject { - Object create(); - } + // case 0: sparse, low-cardinality bitmap + for (int i = 0; i < 100; i++) { + bitmaps[0].add(random.nextInt(totalBits)); + } - @Test - public void testHLLCAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - HLLCAggregator aggr = new HLLCAggregator(10); - aggr.aggregate(new HyperLogLogPlusCounter(10)); - return aggr; + // case 1: 20% full bitmap + for (int i = 0; i < totalBits; i++) { + if (random.nextInt(100) < 20) { + bitmaps[1].add(i); } - }); - System.out.println("HLLC: " + est); - } + } - @Test - public void testBigDecimalAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return newBigDecimalAggr(); + // case 2: half full bitmap + for (int i = 0; i < totalBits; i++) { + if (random.nextInt(100) < 50) { + bitmaps[2].add(i); } + } - }); - System.out.println("BigDecimal: " + est); - } + // case 3: 80% full bitmap + for (int i = 0; i < totalBits; i++) { + if (random.nextInt(100) < 80) { + bitmaps[3].add(i); + } + } - private BigDecimalSumAggregator newBigDecimalAggr() { - BigDecimalSumAggregator aggr = new BigDecimalSumAggregator(); - aggr.aggregate(new BigDecimal("12345678901234567890.123456789")); - return aggr; + // case 4: dense, high-cardinality bitmap + for (int i = 0; i < totalBits; i++) { + if (random.nextInt(totalBits) < 100) { + continue; + } + bitmaps[4].add(i); + } } - @Test - public void testLongAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return newLongAggr(); - } - }); - System.out.println("Long: " + est); + enum Settings { + WITHOUT_MEM_HUNGRY, // only test basic aggrs + WITH_HLLC, // basic aggrs + hllc + WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap + WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap } - private LongSumAggregator newLongAggr() { - LongSumAggregator aggr = new LongSumAggregator(); - aggr.aggregate(new LongMutable(10)); - return aggr; + private MeasureAggregator<?>[] createNoMemHungryAggrs() { + LongSumAggregator longSum = new LongSumAggregator(); + longSum.aggregate(new LongMutable(10)); + + DoubleSumAggregator doubleSum = new DoubleSumAggregator(); + doubleSum.aggregate(new DoubleMutable(10)); + + BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator(); + decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789")); + + return new MeasureAggregator[] { longSum, doubleSum, decimalSum }; } - @Test - public void testDoubleAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return newDoubleAggr(); - } - }); - System.out.println("Double: " + est); + private HLLCAggregator createHLLCAggr() { + HLLCAggregator hllcAggregator = new HLLCAggregator(14); + hllcAggregator.aggregate(new HyperLogLogPlusCounter(14)); + return hllcAggregator; } - private DoubleSumAggregator newDoubleAggr() { - DoubleSumAggregator aggr = new DoubleSumAggregator(); - aggr.aggregate(new DoubleMutable(10)); - return aggr; + private BitmapAggregator createBitmapAggr(boolean lowCardinality) { + BitmapCounter counter = new BitmapCounter(); + counter.merge(lowCardinality ? bitmaps[0] : bitmaps[3]); + + BitmapAggregator result = new BitmapAggregator(); + result.aggregate(counter); + return result; } - @Test - public void testByteArraySize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return new byte[10]; - } - }); - System.out.println("byte[10]: " + est); + private MeasureAggregator<?>[] createAggrs(Settings settings) { + List<MeasureAggregator<?>> aggregators = new ArrayList<>(); + aggregators.addAll(Arrays.asList(createNoMemHungryAggrs())); + + switch (settings) { + case WITHOUT_MEM_HUNGRY: + break; + case WITH_HLLC: + aggregators.add(createHLLCAggr()); + break; + case WITH_LOW_CARD_BITMAP: + aggregators.add(createBitmapAggr(true)); + break; + case WITH_HIGH_CARD_BITMAP: + aggregators.add(createBitmapAggr(false)); + break; + } + + return aggregators.toArray(new MeasureAggregator[aggregators.size()]); } @Test - public void testAggregatorArraySize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return new MeasureAggregator[7]; - } - }); - System.out.println("MeasureAggregator[7]: " + est); + public void testEstimateBitmapMemSize() { + BitmapAggregator[] bitmapAggrs = new BitmapAggregator[bitmaps.length]; + for (int i = 0; i < bitmapAggrs.length; i++) { + bitmapAggrs[i] = new BitmapAggregator(); + bitmapAggrs[i].aggregate(bitmaps[i]); + } + + System.out.printf("%-15s %-10s %-10s\n", "cardinality", "estimate", "actual"); + for (BitmapAggregator aggr : bitmapAggrs) { + System.out.printf("%-15d %-10d %-10d\n", + aggr.getState().getCount(), aggr.getMemBytesEstimate(), meter.measureDeep(aggr)); + } } @Test - public void testTreeMapSize() throws InterruptedException { - final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() { - @Override - public int compare(byte[] o1, byte[] o2) { - return Bytes.compareTo(o1, o2); - } - }); - final Random rand = new Random(); - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - byte[] key = new byte[10]; - rand.nextBytes(key); - map.put(key, null); - return null; - } - }); - System.out.println("TreeMap entry: " + (est - 20)); // -20 is to exclude byte[10] + public void testEstimateMemSize() throws InterruptedException { + int scale = Integer.parseInt(System.getProperty("scale", "1")); + scale = Math.max(1, Math.min(10, scale)); + + testSetting(Settings.WITHOUT_MEM_HUNGRY, scale * 100000); + testSetting(Settings.WITH_HLLC, scale * 5000); + testSetting(Settings.WITH_LOW_CARD_BITMAP, scale * 10000); + testSetting(Settings.WITH_HIGH_CARD_BITMAP, scale * 1000); } - @Test - public void testAggregationCacheSize() throws InterruptedException { - final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() { + private void testSetting(Settings settings, int inputCount) { + SortedMap<byte[], Object> map = new TreeMap<>(new Comparator<byte[]>() { @Override public int compare(byte[] o1, byte[] o2) { return Bytes.compareTo(o1, o2); } }); - final Random rand = new Random(); - - long bytesBefore = memLeft(); - byte[] key = null; - MeasureAggregator<?>[] aggrs = null; - for (int i = 0; i < NUM_OF_OBJS; i++) { - key = new byte[10]; - rand.nextBytes(key); - aggrs = new MeasureAggregator[4]; - aggrs[0] = newBigDecimalAggr(); - aggrs[1] = newLongAggr(); - aggrs[2] = newDoubleAggr(); - aggrs[3] = newDoubleAggr(); - map.put(key, aggrs); - } - - long bytesAfter = memLeft(); - - long mapActualSize = bytesBefore - bytesAfter; - long mapExpectSize = GTAggregateScanner.estimateSizeOfAggrCache(key, aggrs, map.size()); - System.out.println("Actual cache size: " + mapActualSize); - System.out.println("Expect cache size: " + mapExpectSize); - } - - private int estimateObjectSize(CreateAnObject factory) throws InterruptedException { - Object[] hold = new Object[NUM_OF_OBJS]; - long bytesBefore = memLeft(); - for (int i = 0; i < hold.length; i++) { - hold[i] = factory.create(); + final int reportInterval = inputCount / 10; + final Stopwatch stopwatch = new Stopwatch(); + long estimateMillis = 0; + long actualMillis = 0; + + System.out.println("Settings: " + settings); + System.out.printf("%15s %15s %15s %15s %15s\n", + "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)", "Actual(ms)"); + + for (int i = 0; i < inputCount; i++) { + byte[] key = new byte[10]; + random.nextBytes(key); + MeasureAggregator[] values = createAggrs(settings); + map.put(key, values); + + if ((i+1) % reportInterval == 0) { + stopwatch.start(); + long estimateBytes = GTAggregateScanner.estimateSizeOfAggrCache(key, values, map.size()); + estimateMillis += stopwatch.elapsedMillis(); + stopwatch.reset(); + + stopwatch.start(); + long actualBytes = meter.measureDeep(map); + actualMillis += stopwatch.elapsedMillis(); + stopwatch.reset(); + + System.out.printf("%,15d %,15d %,15d %,15d %,15d\n", + map.size(), estimateBytes, actualBytes, estimateMillis, actualMillis); + } } + System.out.println("---------------------------------------\n"); - long bytesAfter = memLeft(); - return (int) ((bytesBefore - bytesAfter) / hold.length); - } - - private long memLeft() throws InterruptedException { - Runtime.getRuntime().gc(); - Thread.sleep(500); - return getSystemAvailBytes(); - } - - private long getSystemAvailBytes() { - Runtime runtime = Runtime.getRuntime(); - long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process - long freeMemory = runtime.freeMemory(); // out of the current heap, how much is free - long maxMemory = runtime.maxMemory(); // Max heap VM can use e.g. Xmx setting - long usedMemory = totalMemory - freeMemory; // how much of the current heap the VM is using - long availableMemory = maxMemory - usedMemory; // available memory i.e. Maximum heap size minus the current amount used - return availableMemory; + map = null; + System.gc(); } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-metadata/pom.xml ---------------------------------------------------------------------- diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 142dd33..c95f7f0 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -65,6 +65,11 @@ </dependency> <dependency> + <groupId>com.github.jbellis</groupId> + <artifactId>jamm</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java new file mode 100644 index 0000000..2883923 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java @@ -0,0 +1,104 @@ +package org.apache.kylin.measure; + +import com.google.common.collect.Lists; +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.measure.basic.*; +import org.apache.kylin.measure.bitmap.BitmapAggregator; +import org.apache.kylin.measure.bitmap.BitmapCounter; +import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; +import org.apache.kylin.measure.hllc.HLLCAggregator; +import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; +import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.datatype.DoubleMutable; +import org.apache.kylin.metadata.datatype.LongMutable; +import org.github.jamm.MemoryMeter; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.math.BigDecimal; +import java.util.List; + +public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase { + private static final MemoryMeter meter = new MemoryMeter(); + + @BeforeClass + public static void setUp() throws Exception { + staticCreateTestMetadata(); + } + + @AfterClass + public static void after() throws Exception { + cleanAfterClass(); + } + + private List<? extends MeasureAggregator> basicAggregators() { + LongMutable longVal = new LongMutable(1000); + LongMinAggregator longMin = new LongMinAggregator(); + LongMaxAggregator longMax = new LongMaxAggregator(); + LongSumAggregator longSum = new LongSumAggregator(); + longMin.aggregate(longVal); + longMax.aggregate(longVal); + longSum.aggregate(longVal); + + DoubleMutable doubleVal = new DoubleMutable(1.0); + DoubleMinAggregator doubleMin = new DoubleMinAggregator(); + DoubleMaxAggregator doubleMax = new DoubleMaxAggregator(); + DoubleSumAggregator doubleSum = new DoubleSumAggregator(); + doubleMin.aggregate(doubleVal); + doubleMax.aggregate(doubleVal); + doubleSum.aggregate(doubleVal); + + BigDecimalMinAggregator decimalMin = new BigDecimalMinAggregator(); + BigDecimalMaxAggregator decimalMax = new BigDecimalMaxAggregator(); + BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator(); + BigDecimal decimal = new BigDecimal("12345678901234567890.123456789"); + decimalMin.aggregate(decimal); + decimalMax.aggregate(decimal); + decimalSum.aggregate(decimal); + + return Lists.newArrayList( + longMin, longMax, longSum, + doubleMin, doubleMax, doubleSum, + decimalMin, decimalMax, decimalSum + ); + } + + private String getAggregatorName(Class<? extends MeasureAggregator> clazz) { + if (!clazz.isAnonymousClass()) { + return clazz.getSimpleName(); + } + String[] parts = clazz.getName().split("\\."); + return parts[parts.length - 1]; + } + + @Test + public void testAggregatorEstimate() { + HLLCAggregator hllcAggregator = new HLLCAggregator(14); + hllcAggregator.aggregate(new HyperLogLogPlusCounter(14)); + + BitmapAggregator bitmapAggregator = new BitmapAggregator(); + BitmapCounter bitmapCounter = new BitmapCounter(); + for (int i = 4000; i <= 100000; i += 2) { + bitmapCounter.add(i); + } + bitmapAggregator.aggregate(bitmapCounter); + + ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType("EXTENDED_COLUMN", DataType.getType("extendedcolumn(100)")); + MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator(); + extendedColumnAggregator.aggregate(new ByteArray(100)); + + List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators()); + aggregators.add(hllcAggregator); + aggregators.add(bitmapAggregator); + aggregators.add(extendedColumnAggregator); + + System.out.printf("%40s %10s %10s\n", "Class", "Estimate", "Actual"); + for (MeasureAggregator aggregator : aggregators) { + String clzName = getAggregatorName(aggregator.getClass()); + System.out.printf("%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator)); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 72e4069..caa09ec 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ <dbunit.version>2.5.2</dbunit.version> <h2.version>1.4.192</h2.version> <jetty.version>9.3.10.v20160621</jetty.version> + <jamm.version>0.3.1</jamm.version> <!-- Commons --> <commons-lang.version>2.6</commons-lang.version> @@ -122,6 +123,8 @@ org/apache/kylin/**/tools/**:**/*CLI.java </sonar.jacoco.excludes> + <!-- JVM Args for Testing --> + <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine> </properties> <licenses> @@ -635,6 +638,11 @@ <artifactId>kryo-shaded</artifactId> <version>${kryo.version}</version> </dependency> + <dependency> + <groupId>com.github.jbellis</groupId> + <artifactId>jamm</artifactId> + <version>${jamm.version}</version> + </dependency> <dependency> <groupId>org.apache.curator</groupId> @@ -869,6 +877,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> + <version>2.10</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -1046,6 +1055,30 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-jamm</id> + <goals> + <goal>copy</goal> + </goals> + <phase>generate-test-resources</phase> + <configuration> + <artifactItems> + <artifactItem> + <groupId>com.github.jbellis</groupId> + <artifactId>jamm</artifactId> + <outputDirectory>${project.build.testOutputDirectory}</outputDirectory> + <destFileName>jamm.jar</destFileName> + </artifactItem> + </artifactItems> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <version>2.19.1</version> <configuration> @@ -1067,7 +1100,7 @@ <value>kylin-log4j.properties</value> </property> </systemProperties> - <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine> + <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine}</argLine> </configuration> </plugin>
