KYLIN-1636 HBase scan benchmark and many refactoring
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ea4d369 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ea4d369 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ea4d369 Branch: refs/heads/1.5.x-HBase1.x Commit: 0ea4d369388c9dec1844a29edb1a3b6a94d54a4e Parents: f4bdd5b Author: Yang Li <liy...@apache.org> Authored: Sun May 1 19:20:50 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sun May 1 19:21:09 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 16 ++ .../apache/kylin/common/KylinConfigBase.java | 2 - .../org/apache/kylin/common/util/ClassUtil.java | 18 +- .../kylin/common/util/Log4jConfigurer.java | 12 +- .../java/org/apache/kylin/gridtable/GTInfo.java | 10 +- .../kylin/gridtable/GTScannerBenchmark.java | 212 ---------------- .../kylin/gridtable/GTScannerBenchmark2.java | 236 ------------------ .../gridtable/SortedGTRecordGenerator.java | 242 ------------------- .../apache/kylin/jdbc/util/Log4jConfigurer.java | 22 +- .../java/org/apache/kylin/rest/DebugTomcat.java | 4 +- .../apache/kylin/rest/util/ClasspathUtil.java | 44 ---- .../hbase/cube/HBaseScannerBenchmark.java | 166 +++++++++++++ .../storage/hbase/cube/SimpleHBaseStore.java | 214 ++++++++++++++++ .../cube/v2/filter/MassInValueProviderImpl.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 154 +++++++----- .../storage/hbase/steps/DeprecatedGCStep.java | 3 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 3 +- .../storage/hbase/util/CleanHtableCLI.java | 3 +- .../storage/hbase/util/CubeMigrationCLI.java | 2 +- .../hbase/util/CubeMigrationCheckCLI.java | 3 +- .../kylin/storage/hbase/util/HBaseClean.java | 3 +- .../kylin/storage/hbase/util/HBaseUsage.java | 3 +- .../storage/hbase/util/HbaseStreamingInput.java | 3 +- .../hbase/util/HtableAlterMetadataCLI.java | 3 +- .../kylin/storage/hbase/util/RowCounterCLI.java | 3 +- 25 files changed, 561 insertions(+), 824 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index 5a583f6..e703122 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -35,6 +35,7 @@ import java.util.Properties; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.restclient.RestClient; +import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.Log4jConfigurer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -240,6 +241,21 @@ public class KylinConfig extends KylinConfigBase { return config; } + + public static void setSandboxEnvIfPossible() { + File dir1 = new File("../../kylin/examples/test_case_data/sandbox"); + File dir2 = new File("../kylin/examples/test_case_data/sandbox"); + + if (dir1.exists()) { + logger.info("Setting sandbox env, KYLIN_CONF=" + dir1.getAbsolutePath()); + ClassUtil.addClasspath(dir1.getAbsolutePath()); + System.setProperty(KylinConfig.KYLIN_CONF, dir1.getAbsolutePath()); + } else if (dir2.exists()) { + logger.info("Setting sandbox env, KYLIN_CONF=" + dir2.getAbsolutePath()); + ClassUtil.addClasspath(dir2.getAbsolutePath()); + System.setProperty(KylinConfig.KYLIN_CONF, dir2.getAbsolutePath()); + } + } // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b10f3c7..05211b1 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; -import java.util.Map; import java.util.Properties; import java.util.SortedSet; import java.util.regex.Matcher; @@ -34,7 +33,6 @@ import org.apache.kylin.common.util.CliCommandExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; @SuppressWarnings("serial") http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java index ce1f014..ef89754 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java @@ -30,16 +30,20 @@ import java.util.WeakHashMap; */ public class ClassUtil { - public static void addClasspath(String path) throws Exception { + public static void addClasspath(String path) { System.out.println("Adding path " + path + " to class path"); File file = new File(path); - if (file.exists()) { - URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader(); - Class<URLClassLoader> urlClass = URLClassLoader.class; - Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class }); - method.setAccessible(true); - method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() }); + try { + if (file.exists()) { + URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader(); + Class<URLClassLoader> urlClass = URLClassLoader.class; + Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class }); + method.setAccessible(true); + method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() }); + } + } catch (Exception e) { + throw new RuntimeException(e); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java b/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java index 696aaff..59eaa4b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java @@ -18,25 +18,25 @@ package org.apache.kylin.common.util; +import java.io.InputStream; import java.util.Enumeration; -import org.apache.log4j.ConsoleAppender; -import org.apache.log4j.Level; +import org.apache.commons.io.IOUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; +import org.apache.log4j.PropertyConfigurator; /** * Created by dongli on 11/24/15. */ public class Log4jConfigurer { - private final static String DEFAULT_PATTERN_LAYOUT = "%d{ISO8601} %-5p [%t] %c{2}:%L : %m%n"; private static boolean INITIALIZED = false; public static void initLogger() { if (!INITIALIZED && !isConfigured()) { - org.apache.log4j.BasicConfigurator.configure(new ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT))); - LogManager.getRootLogger().setLevel(Level.DEBUG); + InputStream is = Log4jConfigurer.class.getClassLoader().getResourceAsStream("kylin-log4j.properties"); + PropertyConfigurator.configure(is); + IOUtils.closeQuietly(is); } INITIALIZED = true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java index 85b45cf..349bd3f 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java @@ -71,10 +71,18 @@ public class GTInfo { return colTypes[i]; } + public int getColumnBlockCount() { + return colBlocks.length; + } + + public ImmutableBitSet getColumnBlock(int i) { + return colBlocks[i]; + } + public ImmutableBitSet getPrimaryKey() { return primaryKey; } - + public ImmutableBitSet getAllColumns() { return colAll; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark.java deleted file mode 100644 index 6b285f0..0000000 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.io.IOException; -import java.util.List; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.gridtable.GTInfo.Builder; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - * Benchmark of processing 10 million GTRecords. 5 dimensions of type int4, and 2 measures of type long8. - */ -public class GTScannerBenchmark { - - final GTInfo info; - final SortedGTRecordGenerator gen; - - final ImmutableBitSet dimensions = ImmutableBitSet.valueOf(0, 1, 2, 3, 4); - final ImmutableBitSet metrics = ImmutableBitSet.valueOf(5, 6); - final String[] aggrFuncs = new String[] { "SUM", "SUM" }; - - final long N = 10000000; // 10M - final long genTime; - - public GTScannerBenchmark() { - Builder builder = GTInfo.builder(); - builder.setCodeSystem(new GTSampleCodeSystem()); - DataType tint = DataType.getType("int4"); - DataType tlong = DataType.getType("long8"); - builder.setColumns(tint, tint, tint, tint, tint, tlong, tlong); - builder.setPrimaryKey(ImmutableBitSet.valueOf(0, 1, 2, 3, 4)); - info = builder.build(); - - gen = new SortedGTRecordGenerator(info); - gen.addDimension(10, 4, null); - gen.addDimension(10, 4, null); - gen.addDimension(10, 4, null); - gen.addDimension(10, 4, null); - gen.addDimension(100, 4, null); - gen.addMeasure(8); - gen.addMeasure(8); - - // warm up - long t = System.currentTimeMillis(); - testGenerate(); - genTime = System.currentTimeMillis() - t; - } - - @SuppressWarnings("unused") - public void testGenerate() { - long count = 0; - for (GTRecord rec : gen.generate(N)) { - count++; - } - } - - @Test - public void testAggregate2() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 1)); - } - - @Test - public void testAggregate2_() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 2)); - } - - @Test - public void testAggregate4() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 1, 2, 3)); - } - - @Test - public void testAggregate5() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 1, 2, 3, 4)); - } - - @SuppressWarnings("unused") - private void testAggregate(ImmutableBitSet groupBy) throws IOException { - long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequest(info, null, dimensions, groupBy, metrics, aggrFuncs, null, true, 10); - IGTScanner scanner = req.decorateScanner(gen.generate(N)); - - long count = 0; - for (GTRecord rec : scanner) { - count++; - } - - t = System.currentTimeMillis() - t; - System.out.println(N + " records aggregated to " + count + ", " + calcSpeed(t) + "K rec/sec"); - } - - private int calcSpeed(long t) { - double sec = (double) (t - genTime) / 1000; - return (int) (N / sec / 1000); - } - - @Test - public void testFilter1() throws IOException { - testFilter(eq(col(1), 1, 5, 7)); - } - - @Test - public void testFilter2() throws IOException { - testFilter( // - and( // - gt(col(0), 5), // - eq(col(2), 2, 4))); - } - - @Test - public void testFilter3() throws IOException { - testFilter( // - and( // - gt(col(0), 2), // - eq(col(4), 1, 3, 5, 9, 12, 14, 23, 43, 52, 78, 92), // - or( // - eq(col(1), 2, 4), // - eq(col(2), 2, 4, 5, 9)))); - } - - @SuppressWarnings("unused") - private void testFilter(TupleFilter filter) throws IOException { - long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequest(info, null, info.colAll, filter); - IGTScanner scanner = req.decorateScanner(gen.generate(N)); - - long count = 0; - for (GTRecord rec : scanner) { - count++; - } - - t = System.currentTimeMillis() - t; - System.out.println(N + " records filtered to " + count + ", " + calcSpeed(t) + "K rec/sec"); - } - - private LogicalTupleFilter and(TupleFilter... filters) { - return logical(FilterOperatorEnum.AND, filters); - } - - private LogicalTupleFilter or(TupleFilter... filters) { - return logical(FilterOperatorEnum.OR, filters); - } - - private LogicalTupleFilter logical(FilterOperatorEnum op, TupleFilter[] filters) { - LogicalTupleFilter r = new LogicalTupleFilter(op); - for (TupleFilter f : filters) - r.addChild(f); - return r; - } - - private CompareTupleFilter gt(ColumnTupleFilter col, int v) { - CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); - r.addChild(col); - - int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); - int len = info.getCodeSystem().maxCodeLength(c); - ByteArray bytes = new ByteArray(len); - BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); - r.addChild(new ConstantTupleFilter(bytes)); - - return r; - } - - private CompareTupleFilter eq(ColumnTupleFilter col, int... values) { - CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN); - r.addChild(col); - - List<ByteArray> list = Lists.newArrayList(); - for (int v : values) { - int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); - int len = info.getCodeSystem().maxCodeLength(c); - ByteArray bytes = new ByteArray(len); - BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); - list.add(bytes); - } - r.addChild(new ConstantTupleFilter(list)); - return r; - } - - private ColumnTupleFilter col(int i) { - return new ColumnTupleFilter(info.colRef(i)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark2.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark2.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark2.java deleted file mode 100644 index 23b0d48..0000000 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScannerBenchmark2.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Random; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.gridtable.GTInfo.Builder; -import org.apache.kylin.gridtable.SortedGTRecordGenerator.Randomizer; -import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.junit.Ignore; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - * Benchmark of processing 10 million GTRecords. 5 dimensions of type int4, and 2 measures of type long8. - * - * All the same as GTScannerBenchmark except for the last measure is single-value HLLC - */ -@Ignore -public class GTScannerBenchmark2 { - - final GTInfo info; - final SortedGTRecordGenerator gen; - - final ImmutableBitSet dimensions = ImmutableBitSet.valueOf(0, 1, 2, 3, 4); - final ImmutableBitSet metrics = ImmutableBitSet.valueOf(5, 6); - final String[] aggrFuncs = new String[] { "SUM", "COUNT_DISTINCT" }; - - final long N = 10000000; // 10M - final long genTime; - - public GTScannerBenchmark2() { - Builder builder = GTInfo.builder(); - builder.setCodeSystem(new GTSampleCodeSystem()); - DataType tint = DataType.getType("int4"); - DataType tlong = DataType.getType("long8"); - DataType thllc = DataType.getType("hllc(12)"); - builder.setColumns(tint, tint, tint, tint, tint, tlong, thllc); - builder.setPrimaryKey(ImmutableBitSet.valueOf(0, 1, 2, 3, 4)); - info = builder.build(); - - gen = new SortedGTRecordGenerator(info); - gen.addDimension(10, 4, null); - gen.addDimension(10, 4, null); - gen.addDimension(10, 4, null); - gen.addDimension(10, 4, null); - gen.addDimension(100, 4, null); - gen.addMeasure(8); - gen.addMeasure(8, new Randomizer() { - HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(12); - - @Override - public int fillRandom(Random rand, byte[] array, int offset) { - try { - hllc.clear(); - hllc.add(rand.nextInt()); - ByteBuffer wrap = ByteBuffer.wrap(array, offset, array.length - offset); - hllc.writeRegisters(wrap); - return wrap.position() - offset; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - - // warm up - long t = System.currentTimeMillis(); - testGenerate(); - genTime = System.currentTimeMillis() - t; - } - - @SuppressWarnings("unused") - public void testGenerate() { - long count = 0; - for (GTRecord rec : gen.generate(N)) { - count++; - } - } - - @Test - public void testAggregate2() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 1)); - } - - @Test - public void testAggregate2_() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 2)); - } - - @Test - public void testAggregate4() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 1, 2, 3)); - } - - @Test - public void testAggregate5() throws IOException { - testAggregate(ImmutableBitSet.valueOf(0, 1, 2, 3, 4)); - } - - @SuppressWarnings("unused") - private void testAggregate(ImmutableBitSet groupBy) throws IOException { - long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequest(info, null, dimensions, groupBy, metrics, aggrFuncs, null, true, 10); - IGTScanner scanner = req.decorateScanner(gen.generate(N)); - - long count = 0; - for (GTRecord rec : scanner) { - count++; - } - - t = System.currentTimeMillis() - t; - System.out.println(N + " records aggregated to " + count + ", " + calcSpeed(t) + "K rec/sec"); - } - - private int calcSpeed(long t) { - double sec = (double) (t - genTime) / 1000; - return (int) (N / sec / 1000); - } - - @Test - public void testFilter1() throws IOException { - testFilter(eq(col(1), 1, 5, 7)); - } - - @Test - public void testFilter2() throws IOException { - testFilter( // - and( // - gt(col(0), 5), // - eq(col(2), 2, 4))); - } - - @Test - public void testFilter3() throws IOException { - testFilter( // - and( // - gt(col(0), 2), // - eq(col(4), 1, 3, 5, 9, 12, 14, 23, 43, 52, 78, 92), // - or( // - eq(col(1), 2, 4), // - eq(col(2), 2, 4, 5, 9)))); - } - - @SuppressWarnings("unused") - private void testFilter(TupleFilter filter) throws IOException { - long t = System.currentTimeMillis(); - GTScanRequest req = new GTScanRequest(info, null, info.colAll, filter); - IGTScanner scanner = req.decorateScanner(gen.generate(N)); - - long count = 0; - for (GTRecord rec : scanner) { - count++; - } - - t = System.currentTimeMillis() - t; - System.out.println(N + " records filtered to " + count + ", " + calcSpeed(t) + "K rec/sec"); - } - - private LogicalTupleFilter and(TupleFilter... filters) { - return logical(FilterOperatorEnum.AND, filters); - } - - private LogicalTupleFilter or(TupleFilter... filters) { - return logical(FilterOperatorEnum.OR, filters); - } - - private LogicalTupleFilter logical(FilterOperatorEnum op, TupleFilter[] filters) { - LogicalTupleFilter r = new LogicalTupleFilter(op); - for (TupleFilter f : filters) - r.addChild(f); - return r; - } - - private CompareTupleFilter gt(ColumnTupleFilter col, int v) { - CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); - r.addChild(col); - - int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); - int len = info.getCodeSystem().maxCodeLength(c); - ByteArray bytes = new ByteArray(len); - BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); - r.addChild(new ConstantTupleFilter(bytes)); - - return r; - } - - private CompareTupleFilter eq(ColumnTupleFilter col, int... values) { - CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN); - r.addChild(col); - - List<ByteArray> list = Lists.newArrayList(); - for (int v : values) { - int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); - int len = info.getCodeSystem().maxCodeLength(c); - ByteArray bytes = new ByteArray(len); - BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); - list.add(bytes); - } - r.addChild(new ConstantTupleFilter(list)); - return r; - } - - private ColumnTupleFilter col(int i) { - return new ColumnTupleFilter(info.colRef(i)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/core-cube/src/test/java/org/apache/kylin/gridtable/SortedGTRecordGenerator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SortedGTRecordGenerator.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SortedGTRecordGenerator.java deleted file mode 100644 index d8b5e48..0000000 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SortedGTRecordGenerator.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.BytesUtil; - -import com.google.common.collect.Lists; - -public class SortedGTRecordGenerator { - - private GTInfo info; - private ArrayList<ColSpec> colSpecs = Lists.newArrayList(); - - public SortedGTRecordGenerator(GTInfo info) { - this.info = info; - } - - public void addDimension(long cardinality, int length, Map<Integer, Integer> weights) { - assert cardinality > 0; - ColSpec spec = new ColSpec(); - spec.cardinality = cardinality; - spec.length = length; - spec.weights = weights; - colSpecs.add(spec); - } - - public void addMeasure(int length) { - addMeasure(length, new BytesRandomizer(length)); - } - - public void addMeasure(int length, Randomizer randomizer) { - assert length > 0; - ColSpec spec = new ColSpec(); - spec.length = length; - spec.measureRandomizer = randomizer; - colSpecs.add(spec); - } - - public IGTScanner generate(long nRows) { - validate(); - return new Generator(nRows); - } - - private void validate() { - if (info.getColumnCount() != colSpecs.size()) - throw new IllegalArgumentException(); - for (int i = 0; i < colSpecs.size(); i++) { - ColSpec spec = colSpecs.get(i); - if (info.codeSystem.maxCodeLength(i) < spec.length) - throw new IllegalArgumentException(); - } - } - - private class ColSpec { - int length; - long cardinality; - Map<Integer, Integer> weights; - long weightSum; - Randomizer measureRandomizer; - } - - public interface Randomizer { - int fillRandom(Random rand, byte[] array, int offset); - } - - public static class BytesRandomizer implements Randomizer { - final private byte bytes[]; - - public BytesRandomizer(int len) { - this.bytes = new byte[len]; - } - - @Override - public int fillRandom(Random rand, byte[] array, int offset) { - rand.nextBytes(bytes); - System.arraycopy(bytes, 0, array, offset, bytes.length); - return bytes.length; - } - } - - private class Generator implements IGTScanner { - final long nRows; - final Random rand; - - int counter; - Distribution[] dist; - GTRecord rec; - - public Generator(long nRows) { - this.nRows = nRows; - this.rand = new Random(); - - rec = new GTRecord(info); - dist = new Distribution[colSpecs.size()]; - for (int i = 0; i < colSpecs.size(); i++) { - ColSpec spec = colSpecs.get(i); - rec.set(i, new ByteArray(spec.length)); - dist[i] = new Distribution(spec, 0); - } - - } - - @Override - public Iterator<GTRecord> iterator() { - return new Iterator<GTRecord>() { - - @Override - public boolean hasNext() { - return counter < nRows; - } - - @Override - public GTRecord next() { - for (int i = 0; i < colSpecs.size(); i++) { - ColSpec spec = colSpecs.get(i); - // dimension case - if (spec.cardinality > 0) { - long v = dist[i].next(); - if (v < 0) { - dist[i] = new Distribution(spec, parentLevelCount(i)); - v = dist[i].next(); - } - ByteArray bytes = rec.get(i); - assert bytes.length() == spec.length; - BytesUtil.writeLong(v, bytes.array(), bytes.offset(), bytes.length()); - } - // measure case - else { - int len = spec.measureRandomizer.fillRandom(rand, rec.get(i).array(), 0); - rec.get(i).setLength(len); - } - } - counter++; - return rec; - } - - private long parentLevelCount(int i) { - if (i == 0) - return nRows; - else - return dist[i - 1].leftRowsForCurValue + 1; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - @Override - public void close() throws IOException { - } - - @Override - public GTInfo getInfo() { - return info; - } - - @Override - public int getScannedRowCount() { - return counter; - } - - } - - private class Distribution { - ColSpec spec; - long nRows; - long leftRows; - long leftRowsForCurValue; - int curValue; - - public Distribution(ColSpec spec, long nRows) { - assert spec.cardinality > 0; - - this.spec = spec; - this.nRows = nRows; - this.leftRows = nRows; - this.leftRowsForCurValue = 0; - this.curValue = -1; - - if (spec.weightSum == 0) { - spec.weightSum = spec.cardinality; // all value is weight 1 by default - if (spec.weights != null) { - for (Entry<Integer, Integer> entry : spec.weights.entrySet()) { - spec.weightSum += entry.getValue() - 1; - } - } - } - } - - private long weight(int v) { - if (spec.weights != null && spec.weights.containsKey(v)) - return spec.weights.get(v); - else - return 1; - } - - public long next() { - if (leftRows == 0) - return -1; - - if (leftRowsForCurValue == 0 && curValue < spec.cardinality - 1) { - curValue++; - if (curValue == spec.cardinality - 1) - leftRowsForCurValue = leftRows; - else - leftRowsForCurValue = (long) (nRows * (double) weight(curValue) / (double) spec.weightSum); - } - - leftRowsForCurValue = Math.max(leftRowsForCurValue - 1, 0); - leftRows--; - return curValue; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java index 994baab..2988c11 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java @@ -18,23 +18,26 @@ package org.apache.kylin.jdbc.util; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; import java.util.Enumeration; -import org.apache.log4j.ConsoleAppender; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; +import org.apache.log4j.PropertyConfigurator; /** * Created by dongli on 11/24/15. */ public class Log4jConfigurer { - private final static String DEFAULT_PATTERN_LAYOUT = "%d{ISO8601} %-5p [%t] %c{2}:%L : %m%n"; private static boolean INITIALIZED = false; public static void initLogger() { if (!INITIALIZED && !isConfigured()) { - org.apache.log4j.BasicConfigurator.configure(new ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT))); + InputStream is = Log4jConfigurer.class.getClassLoader().getResourceAsStream("kylin-log4j.properties"); + PropertyConfigurator.configure(is); + closeQuietly(is); } INITIALIZED = true; } @@ -52,4 +55,15 @@ public class Log4jConfigurer { } return false; } + + public static void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException ioe) { + // ignore + } + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java index c0f61d2..7417a05 100644 --- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java +++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java @@ -30,7 +30,6 @@ import org.apache.catalina.startup.Tomcat; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.util.Shell; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.rest.util.ClasspathUtil; public class DebugTomcat { @@ -39,8 +38,7 @@ public class DebugTomcat { System.setProperty("log4j.configuration", "kylin-log4j.properties"); // test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox - ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath()); - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); + KylinConfig.setSandboxEnvIfPossible(); overrideDevJobJarLocations(); System.setProperty("spring.profiles.active", "testing"); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/server/src/main/java/org/apache/kylin/rest/util/ClasspathUtil.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/util/ClasspathUtil.java b/server/src/main/java/org/apache/kylin/rest/util/ClasspathUtil.java deleted file mode 100644 index 074c46b..0000000 --- a/server/src/main/java/org/apache/kylin/rest/util/ClasspathUtil.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.util; - -import java.io.File; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; - -/** - * @author xduo - * - */ -public class ClasspathUtil { - - public static void addClasspath(String path) throws Exception { - File file = new File(path); - - if (file.exists()) { - URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader(); - Class<URLClassLoader> urlClass = URLClassLoader.class; - Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class }); - method.setAccessible(true); - method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() }); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java new file mode 100644 index 0000000..85021cc --- /dev/null +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.hbase.cube; + +import java.io.IOException; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.ImmutableBitSet; +import org.apache.kylin.gridtable.GTInfo; +import org.apache.kylin.gridtable.GTInfo.Builder; +import org.apache.kylin.gridtable.GTRecord; +import org.apache.kylin.gridtable.GTSampleCodeSystem; +import org.apache.kylin.gridtable.GTScanRequest; +import org.apache.kylin.gridtable.IGTScanner; +import org.apache.kylin.gridtable.IGTWriter; +import org.apache.kylin.gridtable.benchmark.SortedGTRecordGenerator; +import org.apache.kylin.metadata.datatype.DataType; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Benchmark of processing 10 million GTRecords. 5 dimensions of type int4, and 2 measures of type long8. + */ +@Ignore +public class HBaseScannerBenchmark { + + static final Logger logger = LoggerFactory.getLogger(HBaseScannerBenchmark.class); + + final GTInfo info; + final SortedGTRecordGenerator gen; + + final ImmutableBitSet dimensions = ImmutableBitSet.valueOf(0, 1, 2, 3, 4); + final ImmutableBitSet metrics = ImmutableBitSet.valueOf(5, 6); + final String[] aggrFuncs = new String[] { "SUM", "SUM" }; + + final long N = 1000000; // 1M, note the limit memory of HBase in sandbox + + final TableName htableName = TableName.valueOf("HBaseScannerBenchmark"); + final SimpleHBaseStore simpleStore; + + public HBaseScannerBenchmark() throws IOException { + Builder builder = GTInfo.builder(); + builder.setCodeSystem(new GTSampleCodeSystem()); + DataType tint = DataType.getType("int4"); + DataType tlong = DataType.getType("long8"); + builder.setColumns(tint, tint, tint, tint, tint, tlong, tlong); + builder.setPrimaryKey(ImmutableBitSet.valueOf(0, 1, 2, 3, 4)); + info = builder.build(); + + gen = new SortedGTRecordGenerator(info); + gen.addDimension(10, 4, null); + gen.addDimension(10, 4, null); + gen.addDimension(10, 4, null); + gen.addDimension(10, 4, null); + gen.addDimension(100, 4, null); + gen.addMeasure(8); + gen.addMeasure(8); + + simpleStore = new SimpleHBaseStore(info, htableName); + buildTable(); + } + + private void buildTable() throws IOException { + IGTWriter builder = simpleStore.rebuild(); + + logger.info("Writing " + N + " records"); + long t = System.currentTimeMillis(); + + long count = 0; + for (GTRecord rec : gen.generate(N)) { + builder.write(rec); + count++; + if (count % 100000 == 0) + logger.info(count + " rows written"); + } + builder.close(); + + t = System.currentTimeMillis() - t; + logger.info(count + " rows written, " + speed(t) + "K row/sec"); + } + + @Test + public void testScan() throws IOException { + int rounds = 5; + + for (int i = 0; i < rounds; i++) { + testScanRaw("Scan raw " + (i + 1) + " of " + rounds); + testScanRecords("Scan records " + (i + 1) + " of " + rounds); + } + } + + @SuppressWarnings("unused") + private void testScanRaw(String msg) throws IOException { + long t = System.currentTimeMillis(); + + IGTScanner scan = simpleStore.scan(new GTScanRequest(info, null, null, null)); + ResultScanner innerScanner = ((SimpleHBaseStore.Reader) scan).getHBaseScanner(); + int count = 0; + for (Result r : innerScanner) { + count++; + } + scan.close(); + + t = System.currentTimeMillis() - t; + logger.info(msg + ", " + count + " rows, " + speed(t) + "K row/sec"); + } + + @SuppressWarnings("unused") + private void testScanRecords(String msg) throws IOException { + long t = System.currentTimeMillis(); + + IGTScanner scan = simpleStore.scan(new GTScanRequest(info, null, null, null)); + int count = 0; + for (GTRecord rec : scan) { + count++; + } + scan.close(); + + if (scan.getScannedRowCount() != count) + throw new IllegalStateException(); + + t = System.currentTimeMillis() - t; + logger.info(msg + ", " + count + " records, " + speed(t) + "K rec/sec"); + } + + private int speed(long t) { + double sec = (double) t / 1000; + return (int) (N / sec / 1000); + } + + @After + public void cleanup() throws IOException { + simpleStore.cleanup(); + } + + public static void main(String[] args) throws IOException { + KylinConfig.setSandboxEnvIfPossible(); + + HBaseScannerBenchmark benchmark = new HBaseScannerBenchmark(); + benchmark.testScan(); + benchmark.cleanup(); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java new file mode 100644 index 0000000..bbdb542 --- /dev/null +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.hbase.cube; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.kv.RowConstants; +import org.apache.kylin.gridtable.GTInfo; +import org.apache.kylin.gridtable.GTRecord; +import org.apache.kylin.gridtable.GTScanRequest; +import org.apache.kylin.gridtable.IGTScanner; +import org.apache.kylin.gridtable.IGTStore; +import org.apache.kylin.gridtable.IGTWriter; +import org.apache.kylin.storage.hbase.HBaseConnection; +import org.apache.kylin.storage.hbase.steps.CubeHTableUtil; + +public class SimpleHBaseStore implements IGTStore { + + static final String CF = "F"; + static final byte[] CF_B = Bytes.toBytes(CF); + static final String COL = "C"; + static final byte[] COL_B = Bytes.toBytes(COL); + static final int ID_LEN = RowConstants.ROWKEY_CUBOIDID_LEN; + + final GTInfo info; + final TableName htableName; + + public SimpleHBaseStore(GTInfo info, TableName htable) { + this.info = info; + this.htableName = htable; + } + + @Override + public GTInfo getInfo() { + return info; + } + + public void cleanup() throws IOException { + CubeHTableUtil.deleteHTable(htableName); + } + + @Override + public IGTWriter rebuild() throws IOException { + CubeHTableUtil.createBenchmarkHTable(htableName, CF); + return new Writer(); + } + + @Override + public IGTWriter append() throws IOException { + return new Writer(); + } + + @Override + public IGTScanner scan(GTScanRequest scanRequest) throws IOException { + return new Reader(); + } + + private class Writer implements IGTWriter { + final HTableInterface table; + final ByteBuffer rowkey = ByteBuffer.allocate(50); + final ByteBuffer value = ByteBuffer.allocate(50); + + Writer() throws IOException { + HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); + table = conn.getTable(htableName); + table.setAutoFlush(false, true); + } + + @Override + public void write(GTRecord rec) throws IOException { + assert info.getColumnBlockCount() == 2; + + rowkey.clear(); + for (int i = 0; i < ID_LEN; i++) { + rowkey.put((byte) 0); + } + rec.exportColumnBlock(0, rowkey); + rowkey.flip(); + + value.clear(); + rec.exportColumnBlock(1, value); + value.flip(); + + Put put = new Put(rowkey); + put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value); + table.put(put); + } + + @Override + public void close() throws IOException { + table.flushCommits(); + table.close(); + } + } + + class Reader implements IGTScanner { + final HTableInterface table; + final ResultScanner scanner; + + int count = 0; + + Reader() throws IOException { + HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); + table = conn.getTable(htableName); + + Scan scan = new Scan(); + scan.addFamily(CF_B); + scan.setCaching(1024); + scan.setCacheBlocks(true); + scanner = table.getScanner(scan); + } + + public ResultScanner getHBaseScanner() { + return scanner; + } + + @Override + public Iterator<GTRecord> iterator() { + return new Iterator<GTRecord>() { + GTRecord next = null; + GTRecord rec = new GTRecord(info); + + @Override + public boolean hasNext() { + if (next != null) + return true; + + try { + Result r = scanner.next(); + if (r != null) { + loadRecord(r); + next = rec; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return next != null; + } + + private void loadRecord(Result r) { + Cell[] cells = r.rawCells(); + Cell cell = cells[0]; + if (Bytes.compareTo(CF_B, 0, CF_B.length, cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) != 0 // + || Bytes.compareTo(COL_B, 0, COL_B.length, cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) != 0) + throw new IllegalStateException(); + + rec.loadCellBlock(0, ByteBuffer.wrap(cell.getRowArray(), cell.getRowOffset() + ID_LEN, cell.getRowLength() - ID_LEN)); + rec.loadCellBlock(1, ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); + } + + @Override + public GTRecord next() { + if (hasNext() == false) + throw new NoSuchElementException(); + + count++; + next = null; + return rec; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public void close() throws IOException { + scanner.close(); + table.close(); + } + + @Override + public GTInfo getInfo() { + return info; + } + + @Override + public int getScannedRowCount() { + return count; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java index 525645d..47a14eb 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java @@ -27,12 +27,12 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.Pair; import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.metadata.filter.UDF.MassInValueProvider; import org.apache.kylin.metadata.filter.function.Functions; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,7 +66,7 @@ public class MassInValueProviderImpl implements MassInValueProvider { try { synchronized (hdfs_caches) { - fileSystem = FileSystem.get(HBaseConfiguration.create()); + fileSystem = FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration()); long modificationTime = fileSystem.getFileStatus(new Path(filterResourceIdentifier)).getModificationTime(); Pair<Long, Set<ByteArray>> cached = hdfs_caches.getIfPresent(filterResourceIdentifier); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index e7e980a..e1821ee 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; @@ -39,6 +38,7 @@ import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.HBaseColumnFamilyDesc; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +78,7 @@ public class CubeHTableUtil { tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString()); - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin admin = new HBaseAdmin(conf); try { @@ -88,56 +88,7 @@ public class CubeHTableUtil { } for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { - HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName()); - cf.setMaxVersions(1); - - String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase(); - - if (cfDesc.isMemoryHungry()) { - cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize()); - } else { - cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize()); - } - - switch (hbaseDefaultCC) { - case "snappy": { - logger.info("hbase will use snappy to compress data"); - cf.setCompressionType(Algorithm.SNAPPY); - break; - } - case "lzo": { - logger.info("hbase will use lzo to compress data"); - cf.setCompressionType(Algorithm.LZO); - break; - } - case "gz": - case "gzip": { - logger.info("hbase will use gzip to compress data"); - cf.setCompressionType(Algorithm.GZ); - break; - } - case "lz4": { - logger.info("hbase will use lz4 to compress data"); - cf.setCompressionType(Algorithm.LZ4); - break; - } - default: { - logger.info("hbase will not user any compression algorithm to compress data"); - cf.setCompressionType(Algorithm.NONE); - } - } - - try { - String encodingStr = kylinConfig.getHbaseDefaultEncoding(); - DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr); - cf.setDataBlockEncoding(encoding); - } catch (Exception e) { - logger.info("hbase will not user any encoding"); - cf.setDataBlockEncoding(DataBlockEncoding.NONE); - } - - cf.setInMemory(false); - cf.setBloomFilterType(BloomType.NONE); + HColumnDescriptor cf = createColumnFamily(kylinConfig, cfDesc.getName(), cfDesc.isMemoryHungry()); tableDesc.addFamily(cf); } @@ -152,12 +103,105 @@ public class CubeHTableUtil { admin.createTable(tableDesc, splitKeys); Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); - } catch (Exception e) { - logger.error("Failed to create HTable", e); - throw e; } finally { admin.close(); } } + + public static void deleteHTable(TableName tableName) throws IOException { + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); + HBaseAdmin admin = new HBaseAdmin(conf); + try { + if (admin.tableExists(tableName)) { + logger.info("disabling hbase table " + tableName); + admin.disableTable(tableName); + logger.info("deleting hbase table " + tableName); + admin.deleteTable(tableName); + } + } finally { + admin.close(); + } + } + + /** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */ + public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException { + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); + HBaseAdmin admin = new HBaseAdmin(conf); + try { + if (admin.tableExists(tableName)) { + logger.info("disabling hbase table " + tableName); + admin.disableTable(tableName); + logger.info("deleting hbase table " + tableName); + admin.deleteTable(tableName); + } + + HTableDescriptor tableDesc = new HTableDescriptor(tableName); + tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); + + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + tableDesc.addFamily(createColumnFamily(kylinConfig, cfName, false)); + + logger.info("creating hbase table " + tableName); + admin.createTable(tableDesc, null); + Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); + logger.info("create hbase table " + tableName + " done."); + } finally { + admin.close(); + } + } + + public static HColumnDescriptor createColumnFamily(KylinConfig kylinConfig, String cfName, boolean isMemoryHungry) { + HColumnDescriptor cf = new HColumnDescriptor(cfName); + cf.setMaxVersions(1); + + if (isMemoryHungry) { + cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize()); + } else { + cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize()); + } + + String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase(); + switch (hbaseDefaultCC) { + case "snappy": { + logger.info("hbase will use snappy to compress data"); + cf.setCompressionType(Algorithm.SNAPPY); + break; + } + case "lzo": { + logger.info("hbase will use lzo to compress data"); + cf.setCompressionType(Algorithm.LZO); + break; + } + case "gz": + case "gzip": { + logger.info("hbase will use gzip to compress data"); + cf.setCompressionType(Algorithm.GZ); + break; + } + case "lz4": { + logger.info("hbase will use lz4 to compress data"); + cf.setCompressionType(Algorithm.LZ4); + break; + } + default: { + logger.info("hbase will not user any compression algorithm to compress data"); + cf.setCompressionType(Algorithm.NONE); + } + } + + try { + String encodingStr = kylinConfig.getHbaseDefaultEncoding(); + DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr); + cf.setDataBlockEncoding(encoding); + } catch (Exception e) { + logger.info("hbase will not use any encoding"); + cf.setDataBlockEncoding(DataBlockEncoding.NONE); + } + + cf.setInMemory(false); + cf.setBloomFilterType(BloomType.NONE); + return cf; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java index 735f967..472b638 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java @@ -38,6 +38,7 @@ import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hbase.util.HiveCmdBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,7 +100,7 @@ public class DeprecatedGCStep extends AbstractExecutable { List<String> oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin admin = null; try { admin = new HBaseAdmin(conf); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index a4a8a35..a1377f2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -35,6 +35,7 @@ import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +70,7 @@ public class MergeGCStep extends AbstractExecutable { List<String> oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin admin = null; try { admin = new HBaseAdmin(conf); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java index 957e04b..1ba96e9 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +53,7 @@ public class CleanHtableCLI extends AbstractHadoopJob { } private void clean() throws IOException { - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java index 8cedc58..5acecbc 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java @@ -117,7 +117,7 @@ public class CubeMigrationCLI { checkAndGetHbaseUrl(); - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); hbaseAdmin = new HBaseAdmin(conf); hdfsFS = FileSystem.get(new Configuration()); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java index d3a85f0..f1fc8e1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java @@ -34,6 +34,7 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,7 +129,7 @@ public class CubeMigrationCheckCLI { this.dstCfg = kylinConfig; this.ifFix = isFix; - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); hbaseAdmin = new HBaseAdmin(conf); issueExistHTables = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java index 621909a..4ce0dac 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +79,7 @@ public class HBaseClean extends AbstractHadoopJob { private void cleanUp() { try { // get all kylin hbase tables - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java index ffb1e25..0a791b7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -42,7 +43,7 @@ public class HBaseUsage { Map<String, List<String>> envs = Maps.newHashMap(); // get all kylin hbase tables - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java index b5c9b1d..deb8da1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -224,7 +225,7 @@ public class HbaseStreamingInput { } private static HConnection getConnection() throws IOException { - return HConnectionManager.createConnection(HBaseConfiguration.create()); + return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration()); } private static String formatTime(long time) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java index 239adcf..2b4a9a7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +71,7 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob { } private void alter() throws IOException { - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); http://git-wip-us.apache.org/repos/asf/kylin/blob/0ea4d369/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java index 0bf2fc9..b1a5ba4 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +61,7 @@ public class RowCounterCLI { System.out.println("start key in binary: " + Bytes.toStringBinary(startKey)); System.out.println("end key in binary: " + Bytes.toStringBinary(endKey)); - Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); Scan scan = new Scan(); scan.setCaching(512);
