[07/50] [abbrv] kylin git commit: add check on kafka topic and broker info
add check on kafka topic and broker info
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d11d019c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d11d019c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d11d019c
Branch: refs/heads/master-cdh5.7
Commit: d11d019c98e11bf058381752d605f842a8eb30b1
Parents: a4ddbbd
Author: shaofengshi
Authored: Mon Dec 12 16:22:15 2016 +0800
Committer: shaofengshi
Committed: Mon Dec 12 16:22:15 2016 +0800
--
.../org/apache/kylin/source/kafka/KafkaConfigManager.java| 8
.../java/org/apache/kylin/source/kafka/util/KafkaClient.java | 5 +
2 files changed, 13 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d11d019c/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
--
diff --git
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
index e76422c..775f052 100644
---
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
+++
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
@@ -191,6 +191,14 @@ public class KafkaConfigManager {
throw new IllegalArgumentException();
}
+if (StringUtils.isEmpty(kafkaConfig.getTopic())) {
+throw new IllegalArgumentException("No topic info");
+}
+
+if (kafkaConfig.getKafkaClusterConfigs() == null ||
kafkaConfig.getKafkaClusterConfigs().size() ==0) {
+throw new IllegalArgumentException("No cluster info");
+}
+
String path = KafkaConfig.concatResourcePath(kafkaConfig.getName());
getStore().putResource(path, kafkaConfig, KafkaConfig.SERIALIZER);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d11d019c/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
--
diff --git
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
index 2a7b0e8..3b970b3 100644
---
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
+++
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
@@ -18,6 +18,7 @@
package org.apache.kylin.source.kafka.util;
import com.google.common.collect.Maps;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.common.PartitionInfo;
@@ -96,6 +97,10 @@ public class KafkaClient {
}
}
}
+
+if (StringUtils.isEmpty(brokers)) {
+throw new IllegalArgumentException("No cluster info in Kafka
config '" + kafkaConfig.getName() + "'");
+}
return brokers;
}
[40/50] [abbrv] kylin git commit: KYLIN-2290 minor improvements on limit
KYLIN-2290 minor improvements on limit
KYLIN-2290 fix compile
KYLIN-2290 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bd4a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bd4a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bd4a9
Branch: refs/heads/master-cdh5.7
Commit: 6f9bd4a9998cff24c97bce0e6d86282f48fef388
Parents: 699a88e
Author: Hongbin Ma
Authored: Fri Dec 16 16:28:18 2016 +0800
Committer: Hongbin Ma
Committed: Mon Dec 19 14:16:00 2016 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 1 +
.../kylin/cube/CubeCapabilityChecker.java | 5 ++
.../kylin/gridtable/GTScanRequestBuilder.java | 2 +-
.../kylin/metadata/realization/SQLDigest.java | 4 +-
.../apache/kylin/storage/StorageContext.java| 74 +++-
.../storage/gtrecord/CubeScanRangePlanner.java | 9 +--
.../gtrecord/GTCubeStorageQueryBase.java| 1 -
.../gtrecord/SequentialCubeTupleIterator.java | 6 +-
.../apache/kylin/query/ITKylinQueryTest.java| 1 -
.../org/apache/kylin/query/KylinTestBase.java | 7 +-
.../kylin/storage/hbase/ITStorageTest.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 4 ++
.../apache/kylin/query/relnode/OLAPContext.java | 6 +-
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../kylin/query/relnode/OLAPLimitRel.java | 6 +-
.../coprocessor/endpoint/CubeVisitService.java | 7 +-
16 files changed, 60 insertions(+), 77 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 610c2af..5153562 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -778,6 +778,7 @@ abstract public class KylinConfigBase implements
Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments",
"true"));
}
+@Deprecated//Limit is good even it's large. This config is meaning less
since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown",
"1"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 38faed9..c45144b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -120,6 +120,11 @@ public class CubeCapabilityChecker {
return result;
}
+if (digest.limitPrecedesAggr) {
+logger.info("Exclude cube " + cube.getName() + " because there's
limit preceding aggregation");
+return result;
+}
+
if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) {
result.influences.add(new CapabilityInfluence() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index f542de1..bcec1f4 100644
---
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -36,7 +36,7 @@ public class GTScanRequestBuilder {
private boolean allowStorageAggregation = true;
private double aggCacheMemThreshold = 0;
private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage
should terminate itself when $storageScanRowNumThreshold cuboid rows are
scanned, and throw exception.
-private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit
working when $toragePushDownLimit aggregated rows are produced.
+private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit
scanning safely when $toragePushDownLimit aggregated rows are produced.
private long startTime = -1;
private long timeout = -1;
private String storageB
[06/50] [abbrv] kylin git commit: KYLIN-2243 use at least 1Mb for TopNCounterSerializer.maxLength()
KYLIN-2243 use at least 1Mb for TopNCounterSerializer.maxLength() Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4ddbbdd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4ddbbdd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4ddbbdd Branch: refs/heads/master-cdh5.7 Commit: a4ddbbdde876d0690dd65ea8549e8d40cfb5fdd1 Parents: 8ffb0e7 Author: shaofengshi Authored: Mon Dec 12 14:34:19 2016 +0800 Committer: shaofengshi Committed: Mon Dec 12 14:34:19 2016 +0800 -- .../java/org/apache/kylin/measure/topn/TopNCounterSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a4ddbbdd/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java index 071e2a2..cef9177 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java @@ -54,7 +54,7 @@ public class TopNCounterSerializer extends DataTypeSerializer
[05/50] [abbrv] kylin git commit: KYLIN-2244 "kylin.job.cuboid.size.memhungry.ratio" shouldn't be applied on measures like TopN
KYLIN-2244 "kylin.job.cuboid.size.memhungry.ratio" shouldn't be applied on
measures like TopN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ffb0e71
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ffb0e71
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ffb0e71
Branch: refs/heads/master-cdh5.7
Commit: 8ffb0e7103d63d2c0f5d093f3afde1a0490eb8a0
Parents: 4408579
Author: shaofengshi
Authored: Mon Dec 12 14:19:55 2016 +0800
Committer: shaofengshi
Committed: Mon Dec 12 14:19:55 2016 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 5 +++
.../kylin/engine/mr/common/CubeStatsReader.java | 36
2 files changed, 20 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ffb0e71/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2b35c70..610c2af 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -261,10 +261,15 @@ abstract public class KylinConfigBase implements
Serializable {
return
Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
+@Deprecated
public double getJobCuboidSizeMemHungryRatio() {
return
Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio",
"0.05"));
}
+public double getJobCuboidSizeCountDistinctRatio() {
+return
Double.parseDouble(getOptional("kylin.cube.size-estimate-countdistinct-ratio",
"0.05"));
+}
+
public String getCubeAlgorithm() {
return getOptional("kylin.cube.algorithm", "auto");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ffb0e71/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 1cf5da6..21af1e6 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -55,6 +55,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -196,41 +197,34 @@ public class CubeStatsReader {
*/
private static double estimateCuboidStorageSize(CubeSegment cubeSegment,
long cuboidId, long rowCount, long baseCuboidId, List
rowKeyColumnLength) {
-int bytesLength = cubeSegment.getRowKeyPreambleSize();
+int rowkeyLength = cubeSegment.getRowKeyPreambleSize();
KylinConfig kylinConf = cubeSegment.getConfig();
long mask = Long.highestOneBit(baseCuboidId);
long parentCuboidIdActualLength = Long.SIZE -
Long.numberOfLeadingZeros(baseCuboidId);
for (int i = 0; i < parentCuboidIdActualLength; i++) {
if ((mask & cuboidId) > 0) {
-bytesLength += rowKeyColumnLength.get(i);
//colIO.getColumnLength(columnList.get(i));
+rowkeyLength += rowKeyColumnLength.get(i);
//colIO.getColumnLength(columnList.get(i));
}
mask = mask >> 1;
}
// add the measure length
-int space = 0;
-boolean isMemoryHungry = false;
+int normalSpace = rowkeyLength;
+int countDistinctSpace = 0;
for (MeasureDesc measureDesc :
cubeSegment.getCubeDesc().getMeasures()) {
-if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
-isMemoryHungry = true;
-}
DataType returnType =
measureDesc.getFunction().getReturnDataType();
-space += returnType.getStorageBytesEstimate();
-}
-bytesLength += space;
-
-double ret = 1.0 * bytesLength * rowCount / (1024L * 1024L);
-if (isMemoryHungry) {
-double cuboidSizeMemHungryRatio =
kylinConf.getJobCuboidSizeMemHungryRatio();
-logger.info("Cube is memory hungry, storage size estimation
multiply " + cuboidSizeMemHungryRatio);
-ret *= cuboidSizeMemHungryRatio;
-} else {
-
[26/50] [abbrv] kylin git commit: KYLIN 1875 update modify alias
KYLIN 1875 update modify alias
Signed-off-by: zhongjian
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f55cc5c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f55cc5c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f55cc5c4
Branch: refs/heads/master-cdh5.7
Commit: f55cc5c446e8878d059019561894cc70bd0ab89b
Parents: e7cc152
Author: chenzhx <346839...@qq.com>
Authored: Wed Dec 14 16:33:31 2016 +0800
Committer: Li Yang
Committed: Thu Dec 15 18:57:36 2016 +0800
--
webapp/app/css/AdminLTE.css | 5 +
webapp/app/js/controllers/modelDataModel.js | 141 ---
webapp/app/js/controllers/modelEdit.js | 8 +-
webapp/app/js/services/tree.js | 3 +
.../app/partials/modelDesigner/data_model.html | 25 ++--
5 files changed, 119 insertions(+), 63 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/css/AdminLTE.css
--
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 6688457..857dbf7 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4744,6 +4744,11 @@ Gradient Background colors
opacity: 1;
filter: alpha(opacity=100);
}
+.model_graph svg{
+ width:1100px !important;
+ height:600px !important;
+}
+
/*
* Misc: print
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/js/controllers/modelDataModel.js
--
diff --git a/webapp/app/js/controllers/modelDataModel.js
b/webapp/app/js/controllers/modelDataModel.js
index a09101b..b6edd43 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -22,7 +22,7 @@ KylinApp.controller('ModelDataModelCtrl', function
($location,$scope, $modal,cub
$scope.modelsManager = modelsManager;
angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
if(!joinTable.alias){
- joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
+joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
}
});
$scope.init = function (){
@@ -110,10 +110,21 @@ KylinApp.controller('ModelDataModelCtrl', function
($location,$scope, $modal,cub
$scope.editLookup = function (lookup) {
$scope.lookupState.editingIndex = lookupList.indexOf(lookup);
$scope.lookupState.editing = true;
-
// Make a copy of model will be editing.
$scope.newLookup = angular.copy(lookup);
+$scope.newLookup.join.pk_type = [];
+$scope.newLookup.join.fk_type = [];
+$scope.newLookup.join.isCompatible=[];
$scope.newLookup.joinTable=VdmUtil.getNameSpaceTopName($scope.newLookup.join.foreign_key[0]);
+angular.forEach($scope.newLookup.join.primary_key,function(pk,index){
+$scope.newLookup.join.pk_type[index] =
TableModel.getColumnType(VdmUtil.removeNameSpace(pk),$scope.newLookup.table);
+$scope.newLookup.join.fk_type[index] =
TableModel.getColumnType(VdmUtil.removeNameSpace($scope.newLookup.join.foreign_key[index]),$scope.aliasTableMap[$scope.newLookup.joinTable]);
+
if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
+ $scope.newLookup.join.isCompatible[index]=false;
+}else{
+ $scope.newLookup.join.isCompatible[index]=true;
+}
+});
$scope.openLookupModal();
};
@@ -129,24 +140,51 @@ KylinApp.controller('ModelDataModelCtrl', function
($location,$scope, $modal,cub
$scope.doneEditLookup = function () {
// Copy edited model to destination model.
-
$scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
-
$scope.aliasName=[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)];
angular.copy($scope.newLookup,
lookupList[$scope.lookupState.editingIndex]);
-angular.forEach(lookupList,function(joinTable){
- $scope.aliasName.push(joinTable.alias);
- $scope.aliasTableMap[joinTable.alias]=joinTable.table;
- // $scope.tableAliasMap[joinTable.alias]=joinTable.table;
-});
+var oldAlias=$scope.aliasName[$scope.lookupState.editingIndex+1];
+var newAlias=$scope.newLookup.alias;
+if(oldAlias!=newAlias){
+ $scope.aliasName[$scope.lookupState.editingIndex+1]=newAlias;
+ for(var i=0;i<$scope.newLookup.join.primary_key.length;i++){
+
$scope.newLookup.join.primary_key[i]=$scope.newLookup.join.primary_key[i].replace(oldAlias+'.',newAlias+'.');
+
[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed
with coprocessor.");
} catch (Exception ex) {
-logger.error("Error deploying coprocessor on " +
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " +
tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
-public static boolean resetCoprocessor(String tableName, HBaseAdmin
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin,
Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc =
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws
IOException {
+private static List resetCoprocessorOnHTables(final Admin
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws
IOException {
List processedTables = Collections.synchronizedList(new
ArrayList());
ExecutorService coprocessorPool =
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List processedTables;
-public ResetCoprocessorWorker(CountDownLatch countDownLatch,
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch
[47/50] [abbrv] kylin git commit: KYLIN-2287 Speed up model and cube list load in Web
KYLIN-2287 Speed up model and cube list load in Web
Signed-off-by: zhongjian
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64a0a594
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64a0a594
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64a0a594
Branch: refs/heads/master-hbase1.x
Commit: 64a0a5943c034ea6f9cdc6d179d8be9da64fe5ac
Parents: f50c0c8
Author: kangkaisen
Authored: Wed Dec 14 20:08:14 2016 +0800
Committer: zhongjian
Committed: Tue Dec 20 15:12:54 2016 +0800
--
webapp/app/js/controllers/cubeSchema.js | 25 +-
webapp/app/js/controllers/cubes.js | 318 ++-
webapp/app/js/controllers/models.js | 54 ++--
webapp/app/js/model/cubeListModel.js| 4 -
webapp/app/js/model/modelsManager.js| 49 +---
webapp/app/partials/cubes/cubes.html| 12 +-
webapp/app/partials/models/models_tree.html | 16 +-
7 files changed, 215 insertions(+), 263 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js
b/webapp/app/js/controllers/cubeSchema.js
index 357b6af..af8ee7c 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -45,18 +45,6 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope,
QueryService, UserServic
$scope.state = {mode: "view"};
}
- var queryParam = {offset: 0, limit: 65535};
-
- CubeService.list(queryParam, function (all_cubes) {
-if($scope.allCubes.length > 0){
- $scope.allCubes.splice(0,$scope.allCubes.length);
-}
-
-for (var i = 0; i < all_cubes.length; i++) {
- $scope.allCubes.push(all_cubes[i].name.toUpperCase());
-}
- });
-
$scope.$watch('cubeMetaFrame', function (newValue, oldValue) {
if(!newValue){
return;
@@ -198,6 +186,19 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope,
QueryService, UserServic
};
$scope.check_cube_info = function(){
+
+var queryParam = {offset: 0, limit: 65535};
+
+CubeService.list(queryParam, function (all_cubes) {
+ if($scope.allCubes.length > 0){
+$scope.allCubes.splice(0,$scope.allCubes.length);
+ }
+
+ for (var i = 0; i < all_cubes.length; i++) {
+$scope.allCubes.push(all_cubes[i].name.toUpperCase());
+ }
+});
+
if(($scope.state.mode === "edit")
&&$scope.cubeMode=="addNewCube"&&($scope.allCubes.indexOf($scope.cubeMetaFrame.name.toUpperCase())
>= 0)){
SweetAlert.swal('Oops...', "The cube named [" +
$scope.cubeMetaFrame.name.toUpperCase() + "] already exists", 'warning');
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js
b/webapp/app/js/controllers/cubes.js
index b672d3f..cbdbb7d 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -79,43 +79,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q,
$routeParams, $location,
return CubeList.list(queryParam).then(function (resp) {
angular.forEach($scope.cubeList.cubes,function(cube,index){
- cube.streaming = false;
- CubeDescService.query({cube_name: cube.name}, {}, function (detail) {
-if (detail.length > 0 && detail[0].hasOwnProperty("name")) {
- cube.detail = detail[0];
-
ModelService.list({projectName:$scope.projectModel.selectedProject,modelName:cube.detail.model_name},
function (_models) {
-if(_models && _models.length){
- for(var i=0;i<=_models.length;i++){
-if(_models[i].name == cube.detail.model_name){
- cube.model = _models[i];
- var factTable = cube.model.fact_table;
- TableService.get({tableName:factTable},function(table){
-if(table && table.source_type == 1){
- cube.streaming = true;
-}
- })
- break;
-}
- }
-}
-
- })
- //cube.model = modelsManager.getModel(cube.detail.model_name);
-
- defer.resolve(cube.detail);
-
-} else {
- SweetAlert.swal('Oops...', "No cube detail info loaded.",
'error');
-}
- }, function (e) {
-if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take
[07/50] [abbrv] kylin git commit: KYLIN-2243 use at least 1Mb for TopNCounterSerializer.maxLength()
KYLIN-2243 use at least 1Mb for TopNCounterSerializer.maxLength() Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4ddbbdd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4ddbbdd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4ddbbdd Branch: refs/heads/master-hbase1.x Commit: a4ddbbdde876d0690dd65ea8549e8d40cfb5fdd1 Parents: 8ffb0e7 Author: shaofengshi Authored: Mon Dec 12 14:34:19 2016 +0800 Committer: shaofengshi Committed: Mon Dec 12 14:34:19 2016 +0800 -- .../java/org/apache/kylin/measure/topn/TopNCounterSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a4ddbbdd/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java index 071e2a2..cef9177 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java @@ -54,7 +54,7 @@ public class TopNCounterSerializer extends DataTypeSerializer
[06/50] [abbrv] kylin git commit: KYLIN-2244 "kylin.job.cuboid.size.memhungry.ratio" shouldn't be applied on measures like TopN
KYLIN-2244 "kylin.job.cuboid.size.memhungry.ratio" shouldn't be applied on
measures like TopN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ffb0e71
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ffb0e71
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ffb0e71
Branch: refs/heads/master-hbase1.x
Commit: 8ffb0e7103d63d2c0f5d093f3afde1a0490eb8a0
Parents: 4408579
Author: shaofengshi
Authored: Mon Dec 12 14:19:55 2016 +0800
Committer: shaofengshi
Committed: Mon Dec 12 14:19:55 2016 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 5 +++
.../kylin/engine/mr/common/CubeStatsReader.java | 36
2 files changed, 20 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ffb0e71/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2b35c70..610c2af 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -261,10 +261,15 @@ abstract public class KylinConfigBase implements
Serializable {
return
Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
+@Deprecated
public double getJobCuboidSizeMemHungryRatio() {
return
Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio",
"0.05"));
}
+public double getJobCuboidSizeCountDistinctRatio() {
+return
Double.parseDouble(getOptional("kylin.cube.size-estimate-countdistinct-ratio",
"0.05"));
+}
+
public String getCubeAlgorithm() {
return getOptional("kylin.cube.algorithm", "auto");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ffb0e71/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 1cf5da6..21af1e6 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -55,6 +55,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -196,41 +197,34 @@ public class CubeStatsReader {
*/
private static double estimateCuboidStorageSize(CubeSegment cubeSegment,
long cuboidId, long rowCount, long baseCuboidId, List
rowKeyColumnLength) {
-int bytesLength = cubeSegment.getRowKeyPreambleSize();
+int rowkeyLength = cubeSegment.getRowKeyPreambleSize();
KylinConfig kylinConf = cubeSegment.getConfig();
long mask = Long.highestOneBit(baseCuboidId);
long parentCuboidIdActualLength = Long.SIZE -
Long.numberOfLeadingZeros(baseCuboidId);
for (int i = 0; i < parentCuboidIdActualLength; i++) {
if ((mask & cuboidId) > 0) {
-bytesLength += rowKeyColumnLength.get(i);
//colIO.getColumnLength(columnList.get(i));
+rowkeyLength += rowKeyColumnLength.get(i);
//colIO.getColumnLength(columnList.get(i));
}
mask = mask >> 1;
}
// add the measure length
-int space = 0;
-boolean isMemoryHungry = false;
+int normalSpace = rowkeyLength;
+int countDistinctSpace = 0;
for (MeasureDesc measureDesc :
cubeSegment.getCubeDesc().getMeasures()) {
-if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
-isMemoryHungry = true;
-}
DataType returnType =
measureDesc.getFunction().getReturnDataType();
-space += returnType.getStorageBytesEstimate();
-}
-bytesLength += space;
-
-double ret = 1.0 * bytesLength * rowCount / (1024L * 1024L);
-if (isMemoryHungry) {
-double cuboidSizeMemHungryRatio =
kylinConf.getJobCuboidSizeMemHungryRatio();
-logger.info("Cube is memory hungry, storage size estimation
multiply " + cuboidSizeMemHungryRatio);
-ret *= cuboidSizeMemHungryRatio;
-} else {
[31/50] [abbrv] kylin git commit: KYLIN-2291 Collect hive table property skip-header-line-count
KYLIN-2291 Collect hive table property skip-header-line-count
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d79d9374
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d79d9374
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d79d9374
Branch: refs/heads/master-hbase1.x
Commit: d79d9374cd8411518ee8eafc7640910826f6838c
Parents: f2377db
Author: Cheng Wang
Authored: Fri Dec 16 17:23:15 2016 +0800
Committer: Li Yang
Committed: Fri Dec 16 17:45:16 2016 +0800
--
.../org/apache/kylin/source/hive/BeelineHiveClient.java | 3 +++
.../java/org/apache/kylin/source/hive/CLIHiveClient.java | 2 +-
.../apache/kylin/source/hive/HiveSourceTableLoader.java | 1 +
.../java/org/apache/kylin/source/hive/HiveTableMeta.java | 4 +++-
.../apache/kylin/source/hive/HiveTableMetaBuilder.java | 11 ++-
5 files changed, 18 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 47b551b..468ccb1 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -193,6 +193,9 @@ public class BeelineHiveClient implements IHiveClient {
if ("numFiles".equals(resultSet.getString(2).trim())) {
builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
}
+if
("skip.header.line.count".equals(resultSet.getString(2).trim())) {
+
builder.setSkipHeaderLineCount(resultSet.getString(3).trim());
+}
}
}
if ("InputFormat:".equals(resultSet.getString(1).trim())) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index 5a17f1f..e8a93bd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -42,7 +42,6 @@ import com.google.common.collect.Lists;
*
*/
public class CLIHiveClient implements IHiveClient {
-
protected HiveConf hiveConf = null;
protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
@@ -109,6 +108,7 @@ public class CLIHiveClient implements IHiveClient {
builder.setOwner(table.getOwner());
builder.setLastAccessTime(table.getLastAccessTime());
builder.setTableType(table.getTableType());
+
builder.setSkipHeaderLineCount(table.getParameters().get("skip.header.line.count"));
return builder.createHiveTableMeta();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 57292dc..77e1084 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -140,6 +140,7 @@ public class HiveSourceTableLoader {
tableExtDesc.addDataSourceProp("total_file_number",
String.valueOf(hiveTableMeta.fileNum));
tableExtDesc.addDataSourceProp("hive_inputFormat",
hiveTableMeta.sdInputFormat);
tableExtDesc.addDataSourceProp("hive_outputFormat",
hiveTableMeta.sdOutputFormat);
+tableExtDesc.addDataSourceProp("skip_header_line_count",
String.valueOf(hiveTableMeta.skipHeaderLineCount));
metaMgr.saveTableExt(tableExtDesc);
metaMgr.saveSourceTable(tableDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
b/source-hive/src/main/java/org/apache/kylin/source
[13/50] [abbrv] kylin git commit: KYLIN-1832 HyperLogLog performance optimization
KYLIN-1832 HyperLogLog performance optimization
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f05404d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f05404d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f05404d5
Branch: refs/heads/master-hbase1.x
Commit: f05404d5576b52c70cf26eb1bccde1c27cd3852f
Parents: 5303651
Author: xiefan46 <958034...@qq.com>
Authored: Fri Dec 9 16:53:04 2016 +0800
Committer: Li Yang
Committed: Wed Dec 14 11:07:42 2016 +0800
--
.../org/apache/kylin/cube/util/CubingUtils.java | 14 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 22 +-
.../benchmark/GTScannerBenchmark2.java | 4 +-
.../gridtable/AggregationCacheMemSizeTest.java | 4 +-
.../metadata/measure/MeasureCodecTest.java | 4 +-
.../org/apache/kylin/measure/MeasureType.java | 2 +-
.../kylin/measure/MeasureTypeFactory.java | 2 +-
.../kylin/measure/hllc/DenseRegister.java | 91 +
.../kylin/measure/hllc/HLLCAggregator.java | 10 +-
.../kylin/measure/hllc/HLLCMeasureType.java | 20 +-
.../kylin/measure/hllc/HLLCSerializer.java | 16 +-
.../measure/hllc/HLLDistinctCountAggFunc.java | 22 +-
.../measure/hllc/HyperLogLogPlusCounter.java| 392 ---
.../measure/hllc/HyperLogLogPlusCounterNew.java | 388 ++
.../measure/hllc/HyperLogLogPlusCounterOld.java | 392 +++
.../org/apache/kylin/measure/hllc/Register.java | 37 ++
.../apache/kylin/measure/hllc/RegisterType.java | 25 ++
.../kylin/measure/hllc/SparseRegister.java | 98 +
.../measure/AggregatorMemEstimateTest.java | 4 +-
.../measure/hll/HyperLogLogCounterOldTest.java | 265 +
.../measure/hll/HyperLogLogCounterTest.java | 265 -
.../measure/hll2/HyperLogLogCounterNewTest.java | 301 ++
.../hll2/NewHyperLogLogBenchmarkTest.java | 288 ++
.../kylin/engine/mr/common/CubeStatsReader.java | 12 +-
.../kylin/engine/mr/common/CubeStatsWriter.java | 6 +-
.../mr/steps/FactDistinctColumnsReducer.java| 8 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 10 +-
.../engine/mr/steps/MergeStatisticsStep.java| 6 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 8 +-
.../steps/FactDistinctColumnsReducerTest.java | 4 +-
.../apache/kylin/engine/spark/SparkCubing.java | 28 +-
.../cardinality/ColumnCardinalityMapper.java| 12 +-
.../cardinality/ColumnCardinalityReducer.java | 12 +-
.../ColumnCardinalityReducerTest.java | 4 +-
34 files changed, 2002 insertions(+), 774 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 413b907..35139a4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -38,7 +38,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
@@ -59,7 +59,7 @@ public class CubingUtils {
private static Logger logger = LoggerFactory.getLogger(CubingUtils.class);
-public static Map sampling(CubeDesc
cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable> streams) {
+public static Map sampling(CubeDesc
cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable> streams) {
final CubeJoinedFlatTableEnrich flatDesc = new
CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
final int rowkeyLength =
cubeDesc.getRowkey().getRowKeyColumns().length;
final List allCuboidIds = new
CuboidScheduler(cubeDesc).getAllCuboidIds();
@@ -84,9 +84,9 @@ public class CubingUtils {
return result;
}
});
-final Map result =
Maps.newHashMapWithExpectedSize(allCuboidIds.size());
+final Map result =
Maps.newHashMapWithExpectedSize(allCuboidIds.size());
for (Long cuboidId : allCuboidIds) {
-result.put(cuboidId, new
HyperLogLogPlusCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
+result.put(cuboidId, new
HyperLogLogPlusCounterN
[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/yang22-hbase1.x 12aa325e9 -> 96774e707 (forced update)
http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger =
LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(),
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[]
{ item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube,
String cubeName, String projectName) throws IOException {
String projectResPath =
ProjectInstance.concatRe
[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96774e70 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96774e70 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96774e70 Branch: refs/heads/yang22-hbase1.x Commit: 96774e707da7c96d46e9437e2304eedfa5a7c9af Parents: 7eb3d7c Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Tue Dec 20 19:49:55 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 417 insertions(+), 472 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HC
[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96774e70 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96774e70 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96774e70 Branch: refs/heads/yang22-cdh5.7 Commit: 96774e707da7c96d46e9437e2304eedfa5a7c9af Parents: 7eb3d7c Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Tue Dec 20 19:49:55 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 417 insertions(+), 472 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HCon
[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
KYLIN-1672 support kylin on cdh 5.7
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ceccf25
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ceccf25
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ceccf25
Branch: refs/heads/yang22-cdh5.7
Commit: 9ceccf25186c283495638e30d7317dcfbfd2d24c
Parents: 96774e7
Author: Lynne Jiang
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu
Committed: Tue Dec 20 19:51:20 2016 +0800
--
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
.../kylin/engine/mr/steps/MockupMapContext.java | 15 +-
examples/test_case_data/sandbox/core-site.xml | 146 +++---
examples/test_case_data/sandbox/hbase-site.xml | 162 ++
examples/test_case_data/sandbox/hdfs-site.xml | 259 ++
examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
examples/test_case_data/sandbox/yarn-site.xml | 496 ++-
pom.xml | 16 +-
server/pom.xml | 36 ++
.../storage/hbase/steps/MockupMapContext.java | 19 +-
tool/pom.xml| 12 +
11 files changed, 428 insertions(+), 1131 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9ceccf25/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/9ceccf25/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
---
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
outKV[0] = key;
outKV[1] = value;
}
+
}
@Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
throw new NotImplementedException();
}
+
@Override
public float getProgress() {
throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
}
@Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
throw new NotImplementedException();
}
@Override
-public String getJar() {
+public RawComparator getSortComparator() {
throw new NotImplementedException();
}
@Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
throw new NotImplementedException();
}
@@ -221,7 +223,7 @@ public class MockupMapContext {
@Override
public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
}
@Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
public RawComparator getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
});
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9ceccf25/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+
-
fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
-
fs.trash.interval
-360
+1
-
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+
org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apa
[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/yang22-cdh5.7 ab372aaa9 -> 9ceccf251 (forced update)
http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger =
LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(),
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[]
{ item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube,
String cubeName, String projectName) throws IOException {
String projectResPath =
ProjectInstance.concatReso
[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2
KYLIN-2233 Support HBase 1.0.2
Signed-off-by: Yang Li
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8efe45d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8efe45d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8efe45d8
Branch: refs/heads/yang22-hbase102
Commit: 8efe45d8785cedf264748a042eb55e7d1f8b9018
Parents: 96774e7
Author: Cheng Wang
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu
Committed: Tue Dec 20 19:53:01 2016 +0800
--
pom.xml | 2 +-
.../storage/hbase/cube/v1/RegionScannerAdapter.java | 13 +++--
.../coprocessor/observer/AggregateRegionObserver.java | 4 ++--
.../v1/coprocessor/observer/AggregationScanner.java | 14 --
.../observer/ObserverAggregationCache.java| 10 ++
.../observer/AggregateRegionObserverTest.java | 10 ++
.../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 3 +--
7 files changed, 15 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8efe45d8/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
1.2.1
-1.1.1
+1.0.2
0.10.0.0
http://git-wip-us.apache.org/repos/asf/kylin/blob/8efe45d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
-public boolean next(List result, ScannerContext scannerContext)
throws IOException {
+public boolean next(List result, int limit) throws IOException {
return next(result);
}
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
-public boolean nextRaw(List result, ScannerContext scannerContext)
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
return next(result);
}
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
public long getMvccReadPoint() {
return Long.MAX_VALUE;
}
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8efe45d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends
BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterat
[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/yang22-hbase102 a3a91eac1 -> 8efe45d87 (forced update)
http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger =
LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(),
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[]
{ item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube,
String cubeName, String projectName) throws IOException {
String projectResPath =
ProjectInstance.concatRe
[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96774e70 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96774e70 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96774e70 Branch: refs/heads/yang22-hbase102 Commit: 96774e707da7c96d46e9437e2304eedfa5a7c9af Parents: 7eb3d7c Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Tue Dec 20 19:49:55 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 417 insertions(+), 472 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/96774e70/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HC
[5/7] kylin git commit: bug fix
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/473f77b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/473f77b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/473f77b0
Branch: refs/heads/yang22-hbase1.x
Commit: 473f77b06a1554aedbfcfd4fc5b827087aadb426
Parents: 83f9dc7
Author: Hongbin Ma
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma
Committed: Sun Dec 25 14:51:08 2016 +0800
--
.../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/473f77b0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator
{
}
convertCurrentRow(tuple);
return true;
-} finally {
+} catch (Exception e) {
try {
if (cursor != null) {
cursor.close();
}
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
}
+throw e;
}
}
[2/7] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log
introduce CheckUtil to make multiple condition checks clearer in log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e18274c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e18274c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e18274c
Branch: refs/heads/yang22-hbase1.x
Commit: 6e18274c150e21fc952866553e0fd22139477bf1
Parents: 8596ccf
Author: Hongbin Ma
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 16:52:19 2016 +0800
--
.../org/apache/kylin/common/util/CheckUtil.java | 33
.../apache/kylin/rest/service/QueryService.java | 17 ++
2 files changed, 44 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger =
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message,
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() &&
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled =
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query
cache disabled in BackdoorToggles");
+
if (queryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
}
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() -
startTime);
logger.info("Stats of SQL response: isException: {},
duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()),
String.valueOf(sqlResponse.getDuration()),
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold ||
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() <
kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
+if (checkCondit
[7/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49b8d186 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49b8d186 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49b8d186 Branch: refs/heads/yang22-hbase1.x Commit: 49b8d1865b954a4395fe5965988be9297a553454 Parents: 473f77b Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Sun Dec 25 20:13:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 417 insertions(+), 472 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HC
[3/7] kylin git commit: strenthen storage visit deadline calculation
strenthen storage visit deadline calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bb13829
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bb13829
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bb13829
Branch: refs/heads/yang22-hbase1.x
Commit: 6bb1382970f8581b3f8a666ac98829e2a501ffd6
Parents: 6e18274
Author: Hongbin Ma
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 17:57:55 2016 +0800
--
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6bb13829/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() +
scanReq.getTimeout();
logger.info("deadline is " + deadline);
final long storagePushDownLimit =
scanReq.getStoragePushDownLimit();
[4/7] kylin git commit: close OLAPEnumerator at exception
close OLAPEnumerator at exception
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f9dc7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f9dc7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f9dc7a
Branch: refs/heads/yang22-hbase1.x
Commit: 83f9dc7a68e64c8c71dd3dea971f9f0311a5575c
Parents: 6bb1382
Author: Hongbin Ma
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 18:19:02 2016 +0800
--
.../kylin/query/enumerator/OLAPEnumerator.java | 32 +---
1 file changed, 21 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f9dc7a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator
{
@Override
public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
}
-convertCurrentRow(tuple);
-return true;
}
private Object[] convertCurrentRow(ITuple tuple) {
[6/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger =
LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(),
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[]
{ item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube,
String cubeName, String projectName) throws IOException {
String projectResPath =
ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class C
[1/7] kylin git commit: KYLIN-2318 query cache is not working [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/yang22-hbase1.x 25a2f9acf -> 49b8d1865 (forced update)
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8596ccf2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8596ccf2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8596ccf2
Branch: refs/heads/yang22-hbase1.x
Commit: 8596ccf262241826cb60baf3a9a3999c39ae0c66
Parents: da5e5d6
Author: Hongbin Ma
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 15:31:22 2016 +0800
--
.../org/apache/kylin/common/QueryContext.java | 61
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +---
.../apache/kylin/rest/service/QueryService.java | 23
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to
SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
public static Pair getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v ==
[2/8] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log
introduce CheckUtil to make multiple condition checks clearer in log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e18274c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e18274c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e18274c
Branch: refs/heads/yang22-cdh5.7
Commit: 6e18274c150e21fc952866553e0fd22139477bf1
Parents: 8596ccf
Author: Hongbin Ma
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 16:52:19 2016 +0800
--
.../org/apache/kylin/common/util/CheckUtil.java | 33
.../apache/kylin/rest/service/QueryService.java | 17 ++
2 files changed, 44 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger =
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message,
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() &&
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled =
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query
cache disabled in BackdoorToggles");
+
if (queryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
}
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() -
startTime);
logger.info("Stats of SQL response: isException: {},
duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()),
String.valueOf(sqlResponse.getDuration()),
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold ||
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() <
kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
+if (checkConditio
[8/8] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
KYLIN-1672 support kylin on cdh 5.7
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7d00933
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7d00933
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7d00933
Branch: refs/heads/yang22-cdh5.7
Commit: d7d009339ce4d916ec93b005573e22a5c2c0aef1
Parents: 49b8d18
Author: Lynne Jiang
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu
Committed: Sun Dec 25 20:15:17 2016 +0800
--
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
.../kylin/engine/mr/steps/MockupMapContext.java | 15 +-
examples/test_case_data/sandbox/core-site.xml | 146 +++---
examples/test_case_data/sandbox/hbase-site.xml | 162 ++
examples/test_case_data/sandbox/hdfs-site.xml | 259 ++
examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
examples/test_case_data/sandbox/yarn-site.xml | 496 ++-
pom.xml | 16 +-
server/pom.xml | 36 ++
.../storage/hbase/steps/MockupMapContext.java | 19 +-
tool/pom.xml| 12 +
11 files changed, 428 insertions(+), 1131 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7d00933/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7d00933/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
---
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
outKV[0] = key;
outKV[1] = value;
}
+
}
@Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
throw new NotImplementedException();
}
+
@Override
public float getProgress() {
throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
}
@Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
throw new NotImplementedException();
}
@Override
-public String getJar() {
+public RawComparator getSortComparator() {
throw new NotImplementedException();
}
@Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
throw new NotImplementedException();
}
@@ -221,7 +223,7 @@ public class MockupMapContext {
@Override
public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
}
@Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
public RawComparator getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
});
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7d00933/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+
-
fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
-
fs.trash.interval
-360
+1
-
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+
org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apa
[6/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger =
LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(),
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[]
{ item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube,
String cubeName, String projectName) throws IOException {
String projectResPath =
ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class C
[5/8] kylin git commit: bug fix
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/473f77b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/473f77b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/473f77b0
Branch: refs/heads/yang22-cdh5.7
Commit: 473f77b06a1554aedbfcfd4fc5b827087aadb426
Parents: 83f9dc7
Author: Hongbin Ma
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma
Committed: Sun Dec 25 14:51:08 2016 +0800
--
.../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/473f77b0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator
{
}
convertCurrentRow(tuple);
return true;
-} finally {
+} catch (Exception e) {
try {
if (cursor != null) {
cursor.close();
}
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
}
+throw e;
}
}
[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49b8d186 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49b8d186 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49b8d186 Branch: refs/heads/yang22-cdh5.7 Commit: 49b8d1865b954a4395fe5965988be9297a553454 Parents: 473f77b Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Sun Dec 25 20:13:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 417 insertions(+), 472 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HCon
[3/8] kylin git commit: strenthen storage visit deadline calculation
strenthen storage visit deadline calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bb13829
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bb13829
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bb13829
Branch: refs/heads/yang22-cdh5.7
Commit: 6bb1382970f8581b3f8a666ac98829e2a501ffd6
Parents: 6e18274
Author: Hongbin Ma
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 17:57:55 2016 +0800
--
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6bb13829/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() +
scanReq.getTimeout();
logger.info("deadline is " + deadline);
final long storagePushDownLimit =
scanReq.getStoragePushDownLimit();
[4/8] kylin git commit: close OLAPEnumerator at exception
close OLAPEnumerator at exception
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f9dc7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f9dc7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f9dc7a
Branch: refs/heads/yang22-cdh5.7
Commit: 83f9dc7a68e64c8c71dd3dea971f9f0311a5575c
Parents: 6bb1382
Author: Hongbin Ma
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 18:19:02 2016 +0800
--
.../kylin/query/enumerator/OLAPEnumerator.java | 32 +---
1 file changed, 21 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f9dc7a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator
{
@Override
public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
}
-convertCurrentRow(tuple);
-return true;
}
private Object[] convertCurrentRow(ITuple tuple) {
[1/8] kylin git commit: KYLIN-2318 query cache is not working [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/yang22-cdh5.7 68fda3f72 -> d7d009339 (forced update)
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8596ccf2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8596ccf2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8596ccf2
Branch: refs/heads/yang22-cdh5.7
Commit: 8596ccf262241826cb60baf3a9a3999c39ae0c66
Parents: da5e5d6
Author: Hongbin Ma
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 15:31:22 2016 +0800
--
.../org/apache/kylin/common/QueryContext.java | 61
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +---
.../apache/kylin/rest/service/QueryService.java | 23
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to
SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
public static Pair getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v == nul
[2/8] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log
introduce CheckUtil to make multiple condition checks clearer in log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e18274c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e18274c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e18274c
Branch: refs/heads/yang22-hbase102
Commit: 6e18274c150e21fc952866553e0fd22139477bf1
Parents: 8596ccf
Author: Hongbin Ma
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 16:52:19 2016 +0800
--
.../org/apache/kylin/common/util/CheckUtil.java | 33
.../apache/kylin/rest/service/QueryService.java | 17 ++
2 files changed, 44 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger =
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message,
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() &&
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled =
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query
cache disabled in BackdoorToggles");
+
if (queryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
}
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() -
startTime);
logger.info("Stats of SQL response: isException: {},
duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()),
String.valueOf(sqlResponse.getDuration()),
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold ||
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() <
kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
+if (checkCondit
[4/8] kylin git commit: close OLAPEnumerator at exception
close OLAPEnumerator at exception
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f9dc7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f9dc7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f9dc7a
Branch: refs/heads/yang22-hbase102
Commit: 83f9dc7a68e64c8c71dd3dea971f9f0311a5575c
Parents: 6bb1382
Author: Hongbin Ma
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 18:19:02 2016 +0800
--
.../kylin/query/enumerator/OLAPEnumerator.java | 32 +---
1 file changed, 21 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f9dc7a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator
{
@Override
public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
}
-convertCurrentRow(tuple);
-return true;
}
private Object[] convertCurrentRow(ITuple tuple) {
[8/8] kylin git commit: KYLIN-2233 Support HBase 1.0.2
KYLIN-2233 Support HBase 1.0.2
Signed-off-by: Yang Li
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36f64d6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36f64d6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36f64d6a
Branch: refs/heads/yang22-hbase102
Commit: 36f64d6a1c08d2b7bea8cb6299f8fe732239b599
Parents: 49b8d18
Author: Cheng Wang
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu
Committed: Sun Dec 25 20:17:11 2016 +0800
--
pom.xml | 2 +-
.../storage/hbase/cube/v1/RegionScannerAdapter.java | 13 +++--
.../coprocessor/observer/AggregateRegionObserver.java | 4 ++--
.../v1/coprocessor/observer/AggregationScanner.java | 14 --
.../observer/ObserverAggregationCache.java| 10 ++
.../observer/AggregateRegionObserverTest.java | 10 ++
.../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 3 +--
7 files changed, 15 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/36f64d6a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
1.2.1
-1.1.1
+1.0.2
0.10.0.0
http://git-wip-us.apache.org/repos/asf/kylin/blob/36f64d6a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
-public boolean next(List result, ScannerContext scannerContext)
throws IOException {
+public boolean next(List result, int limit) throws IOException {
return next(result);
}
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
-public boolean nextRaw(List result, ScannerContext scannerContext)
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
return next(result);
}
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
public long getMvccReadPoint() {
return Long.MAX_VALUE;
}
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/36f64d6a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends
BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterat
[5/8] kylin git commit: bug fix
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/473f77b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/473f77b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/473f77b0
Branch: refs/heads/yang22-hbase102
Commit: 473f77b06a1554aedbfcfd4fc5b827087aadb426
Parents: 83f9dc7
Author: Hongbin Ma
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma
Committed: Sun Dec 25 14:51:08 2016 +0800
--
.../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/473f77b0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator
{
}
convertCurrentRow(tuple);
return true;
-} finally {
+} catch (Exception e) {
try {
if (cursor != null) {
cursor.close();
}
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
}
+throw e;
}
}
[1/8] kylin git commit: KYLIN-2318 query cache is not working [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/yang22-hbase102 c0da9e250 -> 36f64d6a1 (forced update)
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8596ccf2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8596ccf2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8596ccf2
Branch: refs/heads/yang22-hbase102
Commit: 8596ccf262241826cb60baf3a9a3999c39ae0c66
Parents: da5e5d6
Author: Hongbin Ma
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 15:31:22 2016 +0800
--
.../org/apache/kylin/common/QueryContext.java | 61
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +---
.../apache/kylin/rest/service/QueryService.java | 23
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to
SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
public static Pair getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v ==
[6/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger =
LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn =
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(),
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[]
{ item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube,
String cubeName, String projectName) throws IOException {
String projectResPath =
ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class C
[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49b8d186 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49b8d186 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49b8d186 Branch: refs/heads/yang22-hbase102 Commit: 49b8d1865b954a4395fe5965988be9297a553454 Parents: 473f77b Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Sun Dec 25 20:13:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../kylin/storage/hbase/HBaseStorage.java | 3 +- .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +-- .../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +- .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++- .../cube/v1/SerializedHBaseTupleIterator.java | 4 +- .../observer/AggregateRegionObserver.java | 4 +- .../observer/AggregationScanner.java| 14 ++- .../observer/ObserverAggregationCache.java | 10 ++- .../coprocessor/observer/ObserverEnabler.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../observer/AggregateRegionObserverTest.java | 26 ++ .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 50 files changed, 417 insertions(+), 472 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HC
[3/8] kylin git commit: strenthen storage visit deadline calculation
strenthen storage visit deadline calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bb13829
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bb13829
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bb13829
Branch: refs/heads/yang22-hbase102
Commit: 6bb1382970f8581b3f8a666ac98829e2a501ffd6
Parents: 6e18274
Author: Hongbin Ma
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma
Committed: Sat Dec 24 17:57:55 2016 +0800
--
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6bb13829/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() +
scanReq.getTimeout();
logger.info("deadline is " + deadline);
final long storagePushDownLimit =
scanReq.getStoragePushDownLimit();
kylin git commit: minor, get config by prefix in KylinConfigCLI
Repository: kylin
Updated Branches:
refs/heads/yang22 fd1ed2ed2 -> 6762fae07
minor, get config by prefix in KylinConfigCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6762fae0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6762fae0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6762fae0
Branch: refs/heads/yang22
Commit: 6762fae07f157d762bcafab6996b23dc35c91171
Parents: fd1ed2e
Author: lidongsjtu
Authored: Wed Dec 28 22:17:35 2016 +0800
Committer: lidongsjtu
Committed: Wed Dec 28 22:22:26 2016 +0800
--
.../test_case_data/localmeta/kylin.properties | 3 +
.../org/apache/kylin/tool/KylinConfigCLI.java | 30 +++-
.../apache/kylin/tool/KylinConfigCLITest.java | 79
3 files changed, 108 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6762fae0/examples/test_case_data/localmeta/kylin.properties
--
diff --git a/examples/test_case_data/localmeta/kylin.properties
b/examples/test_case_data/localmeta/kylin.properties
index 1dfac32..f4c6772 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -131,3 +131,6 @@ kylin.test.bcc.new.key=some-value
kylin.engine.mr.config-override.test1=test1
kylin.engine.mr.config-override.test2=test2
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
+
+kylin.engine.provider.0=org.apache.kylin.engine.mr.MRBatchCubingEngine
+kylin.cube.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6762fae0/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
index e1a5b99..b740beb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -18,11 +18,14 @@
package org.apache.kylin.tool;
+import java.util.Map;
import java.util.Properties;
import org.apache.kylin.common.BackwardCompatibilityConfig;
import org.apache.kylin.common.KylinConfig;
+import com.google.common.collect.Maps;
+
public class KylinConfigCLI {
public static void main(String[] args) {
if (args.length != 1) {
@@ -33,10 +36,29 @@ public class KylinConfigCLI {
Properties config = KylinConfig.getKylinProperties();
BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
-String value = config.getProperty(bcc.check(args[0]));
-if (value == null) {
-value = "";
+String key = bcc.check(args[0].trim());
+if (!key.endsWith(".")) {
+String value = config.getProperty(key);
+if (value == null) {
+value = "";
+}
+System.out.println(value);
+} else {
+Map props = getPropertiesByPrefix(config, key);
+for (Map.Entry prop : props.entrySet()) {
+System.out.println(prop.getKey() + "=" + prop.getValue());
+}
+}
+}
+
+static private Map getPropertiesByPrefix(Properties props,
String prefix) {
+Map result = Maps.newLinkedHashMap();
+for (Map.Entry entry : props.entrySet()) {
+String entryKey = (String) entry.getKey();
+if (entryKey.startsWith(prefix)) {
+result.put(entryKey.substring(prefix.length()), (String)
entry.getValue());
+}
}
-System.out.println(value);
+return result;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6762fae0/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
--
diff --git a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
new file mode 100644
index 000..5c56b5f
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ *
+ * http://kyligence.io
+ *
+ * This software is the confidential and proprietary information of
+ * Kyligence Inc. ("Confidential Information"). You shall not disclose
+ * such Confidential Information and shall use it only in accordance
+ * with the terms of the license agreement you entered into with
+ * Kyligence Inc.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED T
kylin git commit: minor, fix wrong header
Repository: kylin
Updated Branches:
refs/heads/yang22 6762fae07 -> 17e323eed
minor, fix wrong header
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/17e323ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/17e323ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/17e323ee
Branch: refs/heads/yang22
Commit: 17e323eed9b6b2ef91b1badf230239280ff6d46d
Parents: 6762fae
Author: lidongsjtu
Authored: Wed Dec 28 22:25:44 2016 +0800
Committer: lidongsjtu
Committed: Wed Dec 28 22:25:52 2016 +0800
--
.../apache/kylin/tool/KylinConfigCLITest.java | 34
1 file changed, 14 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/17e323ee/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
--
diff --git a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
index 5c56b5f..7d1e248 100644
--- a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
+++ b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
@@ -1,26 +1,20 @@
/*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://kyligence.io
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- * such Confidential Information and shall use it only in accordance
- * with the terms of the license agreement you entered into with
- * Kyligence Inc.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.kylin.tool;
kylin git commit: minor, get config by prefix in KylinConfigCLI
Repository: kylin
Updated Branches:
refs/heads/master 08eda7b71 -> fdecf0937
minor, get config by prefix in KylinConfigCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdecf093
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdecf093
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdecf093
Branch: refs/heads/master
Commit: fdecf0937802a2d17418595e6c0543c9668ac8a4
Parents: 08eda7b
Author: lidongsjtu
Authored: Wed Dec 28 22:17:35 2016 +0800
Committer: lidongsjtu
Committed: Wed Dec 28 22:45:57 2016 +0800
--
.../test_case_data/localmeta/kylin.properties | 3 +
.../org/apache/kylin/tool/KylinConfigCLI.java | 30 ++--
.../apache/kylin/tool/KylinConfigCLITest.java | 73
3 files changed, 102 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/examples/test_case_data/localmeta/kylin.properties
--
diff --git a/examples/test_case_data/localmeta/kylin.properties
b/examples/test_case_data/localmeta/kylin.properties
index 1dfac32..f4c6772 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -131,3 +131,6 @@ kylin.test.bcc.new.key=some-value
kylin.engine.mr.config-override.test1=test1
kylin.engine.mr.config-override.test2=test2
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
+
+kylin.engine.provider.0=org.apache.kylin.engine.mr.MRBatchCubingEngine
+kylin.cube.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
index e1a5b99..b740beb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -18,11 +18,14 @@
package org.apache.kylin.tool;
+import java.util.Map;
import java.util.Properties;
import org.apache.kylin.common.BackwardCompatibilityConfig;
import org.apache.kylin.common.KylinConfig;
+import com.google.common.collect.Maps;
+
public class KylinConfigCLI {
public static void main(String[] args) {
if (args.length != 1) {
@@ -33,10 +36,29 @@ public class KylinConfigCLI {
Properties config = KylinConfig.getKylinProperties();
BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
-String value = config.getProperty(bcc.check(args[0]));
-if (value == null) {
-value = "";
+String key = bcc.check(args[0].trim());
+if (!key.endsWith(".")) {
+String value = config.getProperty(key);
+if (value == null) {
+value = "";
+}
+System.out.println(value);
+} else {
+Map props = getPropertiesByPrefix(config, key);
+for (Map.Entry prop : props.entrySet()) {
+System.out.println(prop.getKey() + "=" + prop.getValue());
+}
+}
+}
+
+static private Map getPropertiesByPrefix(Properties props,
String prefix) {
+Map result = Maps.newLinkedHashMap();
+for (Map.Entry entry : props.entrySet()) {
+String entryKey = (String) entry.getKey();
+if (entryKey.startsWith(prefix)) {
+result.put(entryKey.substring(prefix.length()), (String)
entry.getValue());
+}
}
-System.out.println(value);
+return result;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
--
diff --git a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
new file mode 100644
index 000..7d1e248
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writ
kylin git commit: minor, fix wrong prop name in kylin.properties
Repository: kylin Updated Branches: refs/heads/yang22 0547a7caf -> a27d29073 minor, fix wrong prop name in kylin.properties Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a27d2907 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a27d2907 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a27d2907 Branch: refs/heads/yang22 Commit: a27d290736b7317c3a4248dba1c942960cd714a6 Parents: 0547a7c Author: lidongsjtu Authored: Thu Dec 29 22:53:45 2016 +0800 Committer: lidongsjtu Committed: Thu Dec 29 22:55:02 2016 +0800 -- build/conf/kylin.properties | 2 +- examples/test_case_data/sandbox/kylin.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/a27d2907/build/conf/kylin.properties -- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 131a725..5e7d235 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -30,7 +30,7 @@ kylin.env=QA ### SERVER | WEB ### # Kylin server mode, valid value [all, query, job] -kyin.server.mode=all +kylin.server.mode=all # List of web servers in use, this enables one web server instance to sync up with other servers. kylin.server.cluster-servers=localhost:7070 http://git-wip-us.apache.org/repos/asf/kylin/blob/a27d2907/examples/test_case_data/sandbox/kylin.properties -- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 20bc427..1724619 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -18,7 +18,7 @@ ### SERVICE ### # Kylin server mode, valid value [all, query, job] -kyin.server.mode=all +kylin.server.mode=all # Optional information for the owner of kylin platform, it can be your team's email # Currently it will be attached to each kylin's htable attribute
kylin git commit: minor, fix wrong prop name in kylin.properties
Repository: kylin Updated Branches: refs/heads/master 020cf5f00 -> 75c8d13c7 minor, fix wrong prop name in kylin.properties Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75c8d13c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75c8d13c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75c8d13c Branch: refs/heads/master Commit: 75c8d13c708e66c995cdea494f8656b72636a4d6 Parents: 020cf5f Author: lidongsjtu Authored: Thu Dec 29 22:53:45 2016 +0800 Committer: lidongsjtu Committed: Thu Dec 29 22:57:54 2016 +0800 -- build/conf/kylin.properties | 2 +- examples/test_case_data/sandbox/kylin.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/75c8d13c/build/conf/kylin.properties -- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 131a725..5e7d235 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -30,7 +30,7 @@ kylin.env=QA ### SERVER | WEB ### # Kylin server mode, valid value [all, query, job] -kyin.server.mode=all +kylin.server.mode=all # List of web servers in use, this enables one web server instance to sync up with other servers. kylin.server.cluster-servers=localhost:7070 http://git-wip-us.apache.org/repos/asf/kylin/blob/75c8d13c/examples/test_case_data/sandbox/kylin.properties -- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 20bc427..1724619 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -18,7 +18,7 @@ ### SERVICE ### # Kylin server mode, valid value [all, query, job] -kyin.server.mode=all +kylin.server.mode=all # Optional information for the owner of kylin platform, it can be your team's email # Currently it will be attached to each kylin's htable attribute
kylin git commit: minor, fix typo
Repository: kylin
Updated Branches:
refs/heads/yang22 a27d29073 -> ab6aecd41
minor, fix typo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab6aecd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab6aecd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab6aecd4
Branch: refs/heads/yang22
Commit: ab6aecd410d8463c149db0feed90433d8793f949
Parents: a27d290
Author: lidongsjtu
Authored: Fri Dec 30 09:49:55 2016 +0800
Committer: lidongsjtu
Committed: Fri Dec 30 09:49:55 2016 +0800
--
build/smoke-test/smoke-test.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ab6aecd4/build/smoke-test/smoke-test.sh
--
diff --git a/build/smoke-test/smoke-test.sh b/build/smoke-test/smoke-test.sh
index f174b47..a71faf9 100755
--- a/build/smoke-test/smoke-test.sh
+++ b/build/smoke-test/smoke-test.sh
@@ -43,7 +43,7 @@ mkdir -p ${TARGET_PATH}
KYLIN_PID=`cat "${TARGET_PATH}/*kylin*/pid"`
if [ -n "${KYLIN_PID}" ]; then
if ps -p ${KYLIN_PID} > /dev/null; then
-echo "Kylin is running, will be killed. (pid=${KYILN_PID})"
+echo "Kylin is running, will be killed. (pid=${KYLIN_PID})"
kill -9 ${KYLIN_PID}
fi
fi
svn commit: r1776525 - in /kylin/site: docs15/install/advance_settings.html docs16/install/advance_settings.html feed.xml
Author: lidong Date: Fri Dec 30 08:06:38 2016 New Revision: 1776525 URL: http://svn.apache.org/viewvc?rev=1776525&view=rev Log: fix wrong mapreduce parameter Modified: kylin/site/docs15/install/advance_settings.html kylin/site/docs16/install/advance_settings.html kylin/site/feed.xml Modified: kylin/site/docs15/install/advance_settings.html URL: http://svn.apache.org/viewvc/kylin/site/docs15/install/advance_settings.html?rev=1776525&r1=1776524&r2=1776525&view=diff == --- kylin/site/docs15/install/advance_settings.html (original) +++ kylin/site/docs15/install/advance_settings.html Fri Dec 30 08:06:38 2016 @@ -2121,8 +2121,8 @@ The conf/kylin_job_conf.xml and conf/kylin_job_conf_inmem.xml manage the default configurations for Hadoop jobs. If you have the need to customize the configs by cube, you can achieve that with the similar way as above, but need adding a prefix kylin.job.mr.config.override.; These configs will be parsed out and then applied when submitting jobs. See two examples below: - If want a cubeâs job getting more memory from Yarn, you can define: kylin.job.mr.config.override.mapred.map.child.java.opts=-Xmx7g and kylin.job.mr.config.override.mapred.map.memory.mb=8192 - If want a cubeâs job going to a different Yarn resource queue, you can define: kylin.job.mr.config.override.mapred.job.queuename=myQueue (note: âmyQueueâ is just a sample) + If want a cubeâs job getting more memory from Yarn, you can define: kylin.job.mr.config.override.mapreduce.map.java.opts=-Xmx7g and kylin.job.mr.config.override.mapreduce.map.memory.mb=8192 + If want a cubeâs job going to a different Yarn resource queue, you can define: kylin.job.mr.config.override.mapreduce.job.queuename=myQueue (note: âmyQueueâ is just a sample) Enable compression Modified: kylin/site/docs16/install/advance_settings.html URL: http://svn.apache.org/viewvc/kylin/site/docs16/install/advance_settings.html?rev=1776525&r1=1776524&r2=1776525&view=diff == --- kylin/site/docs16/install/advance_settings.html (original) +++ kylin/site/docs16/install/advance_settings.html Fri Dec 30 08:06:38 2016 @@ -2103,8 +2103,8 @@ The conf/kylin_job_conf.xml and conf/kylin_job_conf_inmem.xml manage the default configurations for Hadoop jobs. If you have the need to customize the configs by cube, you can achieve that with the similar way as above, but need adding a prefix kylin.job.mr.config.override.; These configs will be parsed out and then applied when submitting jobs. See two examples below: - If want a cubeâs job getting more memory from Yarn, you can define: kylin.job.mr.config.override.mapred.map.child.java.opts=-Xmx7g and kylin.job.mr.config.override.mapred.map.memory.mb=8192 - If want a cubeâs job going to a different Yarn resource queue, you can define: kylin.job.mr.config.override.mapred.job.queuename=myQueue (note: âmyQueueâ is just a sample) + If want a cubeâs job getting more memory from Yarn, you can define: kylin.job.mr.config.override.mapreduce.map.java.opts=-Xmx7g and kylin.job.mr.config.override.mapreduce.map.memory.mb=8192 + If want a cubeâs job going to a different Yarn resource queue, you can define: kylin.job.mr.config.override.mapreduce.job.queuename=myQueue (note: âmyQueueâ is just a sample) Enable compression Modified: kylin/site/feed.xml URL: http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1776525&r1=1776524&r2=1776525&view=diff == --- kylin/site/feed.xml (original) +++ kylin/site/feed.xml Fri Dec 30 08:06:38 2016 @@ -19,8 +19,8 @@ Apache Kylin Home http://kylin.apache.org/ http://kylin.apache.org/feed.xml"; rel="self" type="application/rss+xml"/> -Fri, 16 Dec 2016 05:59:53 -0800 -Fri, 16 Dec 2016 05:59:53 -0800 +Fri, 30 Dec 2016 00:05:31 -0800 +Fri, 30 Dec 2016 00:05:31 -0800 Jekyll v2.5.3
kylin git commit: KYLIN-2340 fix check style
Repository: kylin
Updated Branches:
refs/heads/master 9d3573a66 -> fdcb9b81d
KYLIN-2340 fix check style
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdcb9b81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdcb9b81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdcb9b81
Branch: refs/heads/master
Commit: fdcb9b81d49d81a66cd290837d0dd89f0a308018
Parents: 9d3573a
Author: lidongsjtu
Authored: Fri Dec 30 16:02:34 2016 +0800
Committer: lidongsjtu
Committed: Fri Dec 30 16:02:34 2016 +0800
--
.../apache/kylin/query/relnode/OLAPToEnumerableConverter.java | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdcb9b81/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 405a24e..8f01a74 100644
---
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -42,9 +42,7 @@ import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.ModelChooser;
@@ -82,10 +80,10 @@ public class OLAPToEnumerableConverter extends
ConverterImpl implements Enumerab
// identify model
List contexts = listContextsHavingScan();
Set candidates = ModelChooser.selectModel(contexts);
-
+
// identify realization for each context
for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
-
+
// Context has no table scan is created by OLAPJoinRel which looks
like
// (sub-query) as A join (sub-query) as B
// No realization needed for such context.
[47/50] [abbrv] kylin git commit: minor, setEndTime for discarded but finished job
minor, setEndTime for discarded but finished job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d3573a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d3573a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d3573a6
Branch: refs/heads/master-hbase1.x
Commit: 9d3573a661d98c6a56716edc80e667b4050c54da
Parents: 6e4637c
Author: Billy Liu
Authored: Fri Dec 30 15:55:35 2016 +0800
Committer: Billy Liu
Committed: Fri Dec 30 15:55:48 2016 +0800
--
.../org/apache/kylin/job/execution/DefaultChainedExecutable.java| 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d3573a6/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 253072e..8bcaaad 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -127,6 +127,7 @@ public class DefaultChainedExecutable extends
AbstractExecutable implements Chai
} else if (hasRunning) {
mgr.updateJobOutput(getId(), ExecutableState.RUNNING, null,
null);
} else if (hasDiscarded) {
+setEndTime(System.currentTimeMillis());
mgr.updateJobOutput(getId(), ExecutableState.DISCARDED, null,
null);
} else {
mgr.updateJobOutput(getId(), ExecutableState.READY, null,
null);
[06/50] [abbrv] kylin git commit: KYLIN-2314 more CI fixes
KYLIN-2314 more CI fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed8b1683
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed8b1683
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed8b1683
Branch: refs/heads/master-hbase1.x
Commit: ed8b1683a35488ffb8dd222c7bbe0fa84600aa5d
Parents: 73fbbb9
Author: Yang Li
Authored: Sat Dec 24 21:09:21 2016 +0800
Committer: Yang Li
Committed: Mon Dec 26 07:25:54 2016 +0800
--
examples/test_case_data/localmeta/project/default.json | 9 -
.../main/java/org/apache/kylin/query/schema/OLAPTable.java | 2 +-
.../kylin/source/kafka/config/KafkaConsumerProperties.java | 8 +++-
3 files changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/examples/test_case_data/localmeta/project/default.json
--
diff --git a/examples/test_case_data/localmeta/project/default.json
b/examples/test_case_data/localmeta/project/default.json
index 1ecc7cc..75d7ec9 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -31,11 +31,18 @@
"tables": [
"DEFAULT.TEST_KYLIN_FACT",
"DEFAULT.TEST_ORDER",
+"DEFAULT.TEST_ACCOUNT",
+"DEFAULT.TEST_COUNTRY",
"EDW.TEST_CAL_DT",
"DEFAULT.TEST_CATEGORY_GROUPINGS",
"EDW.TEST_SITES",
"EDW.TEST_SELLER_TYPE_DIM",
-"DEFAULT.STREAMING_TABLE"
+"DEFAULT.STREAMING_TABLE",
+"SSB.CUSTOMER",
+"SSB.DATES",
+"SSB.PART",
+"SSB.SUPPLIER",
+"SSB.V_LINEORDER"
],
"models": [
"ssb",
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index c33c4ed..85996f4 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -203,7 +203,7 @@ public class OLAPTable extends AbstractQueryableTable
implements TranslatableTab
DataModelDesc model = realization.getModel();
for (String metricColumn : model.getMetrics()) {
TblColRef col = model.findColumn(metricColumn);
-if (col.getType().isIntegerFamily() &&
!col.getType().isBigInt())
+if (col.getTable().equals(sourceTable.getIdentity()) &&
col.getType().isIntegerFamily() && !col.getType().isBigInt())
updateColumns.add(col.getColumnDesc());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
--
diff --git
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
index 8f0dd42..5b3dd87 100644
---
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
+++
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
@@ -101,8 +101,8 @@ public class KafkaConsumerProperties {
private Properties loadKafkaConsumerProperties() {
File propFile = getKafkaConsumerFile();
if (propFile == null || !propFile.exists()) {
-logger.error("fail to locate " + KAFKA_CONSUMER_FILE);
-throw new RuntimeException("fail to locate " +
KAFKA_CONSUMER_FILE);
+logger.warn("fail to locate " + KAFKA_CONSUMER_FILE + ", use empty
kafka consumer properties");
+return new Properties();
}
Properties properties = new Properties();
try {
@@ -115,7 +115,6 @@ public class KafkaConsumerProperties {
File propOverrideFile = new File(propFile.getParentFile(),
propFile.getName() + ".override");
if (propOverrideFile.exists()) {
FileInputStream ois = new FileInputStream(propOverrideFile);
-Properties propOverride = new Properties();
Configuration oconf = new Configuration();
oconf.addResource(ois);
properties.putAll(extractKafkaConfigToProperties(oconf));
@@ -134,7 +133,6 @@ public class KafkaConsumerProperties {
}
private File getKafkaConsumerFile() {
-KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String kylinConfHome = System.getProp
[36/50] [abbrv] kylin git commit: KYLIN-2332 Refactor TupleConverter a bit
KYLIN-2332 Refactor TupleConverter a bit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13c34079
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13c34079
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13c34079
Branch: refs/heads/master-hbase1.x
Commit: 13c34079daa0e102b817d15ccfe2d70b8b364c34
Parents: 687d5fd
Author: Li Yang
Authored: Thu Dec 29 12:18:45 2016 +0800
Committer: Li Yang
Committed: Thu Dec 29 13:38:19 2016 +0800
--
.../org/apache/kylin/storage/IStorageQuery.java | 3 -
.../apache/kylin/storage/StorageContext.java| 10
.../storage/gtrecord/CubeSegmentScanner.java| 3 +-
.../storage/gtrecord/CubeTupleConverter.java| 60 +---
.../gtrecord/GTCubeStorageQueryBase.java| 7 ++-
.../kylin/storage/gtrecord/ITupleConverter.java | 30 ++
.../gtrecord/SegmentCubeTupleIterator.java | 8 +--
.../gtrecord/SequentialCubeTupleIterator.java | 55 +-
.../storage/hbase/cube/v2/CubeStorageQuery.java | 1 +
9 files changed, 108 insertions(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
index 6b53b5b..5455cc3 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
@@ -23,9 +23,6 @@ import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.TupleInfo;
/**
- *
- * @author xjiang
- *
*/
public interface IStorageQuery {
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 9ef59fd..ec46f83 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -47,6 +47,7 @@ public class StorageContext {
private boolean limitEnabled = false;
private boolean enableCoprocessor = false;
+private IStorageQuery storageQuery;
private AtomicLong totalScanCount = new AtomicLong();
private Cuboid cuboid;
private boolean partialResultReturned = false;
@@ -193,4 +194,13 @@ public class StorageContext {
public void setReusedPeriod(Range reusedPeriod) {
this.reusedPeriod = reusedPeriod;
}
+
+public IStorageQuery getStorageQuery() {
+return storageQuery;
+}
+
+public void setStorageQuery(IStorageQuery storageQuery) {
+this.storageQuery = storageQuery;
+}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index c6a6daa..9d6f946 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -52,7 +52,7 @@ public class CubeSegmentScanner implements IGTScanner {
final GTScanRequest scanRequest;
public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid,
Set dimensions, Set groups, //
-Collection metrics, TupleFilter originalfilter,
StorageContext context, String gtStorage) {
+Collection metrics, TupleFilter originalfilter,
StorageContext context) {
logger.info("Init CubeSegmentScanner for segment {}",
cubeSeg.getName());
@@ -78,6 +78,7 @@ public class CubeSegmentScanner implements IGTScanner {
throw new RuntimeException(e);
}
scanRequest = scanRangePlanner.planScanRequest();
+String gtStorage = ((GTCubeStorageQueryBase)
context.getStorageQuery()).getGTStorage();
scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/13c34079/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
--
diff --git
a/core-st
[34/50] [abbrv] kylin git commit: KYLIN-2330 fix CubeDesc returning redundant DeriveInfo
KYLIN-2330 fix CubeDesc returning redundant DeriveInfo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a190ecf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a190ecf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a190ecf
Branch: refs/heads/master-hbase1.x
Commit: 1a190ecfcaad88e2551c5aff3bd4938719ba46a6
Parents: fdecf09
Author: Yang Li
Authored: Thu Dec 29 07:13:12 2016 +0800
Committer: Yang Li
Committed: Thu Dec 29 07:31:17 2016 +0800
--
.../org/apache/kylin/common/QueryContext.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 46 +++-
.../org/apache/kylin/cube/CubeDescTest.java | 45 ++-
.../test_case_data/localmeta/cube_desc/ssb.json | 12 -
4 files changed, 91 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index ef0cb14..93b8556 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -23,7 +23,7 @@ import java.util.Map;
import com.google.common.collect.Maps;
/**
- * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for
comparision
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for
comparison
*/
public class QueryContext {
private static final ThreadLocal> _queryContext = new
ThreadLocal>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 3b8d034..4686a15 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -807,25 +807,49 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
for (int i = 0; i < derivedCols.length; i++) {
if (ArrayUtils.contains(hostCols, derivedCols[i])) {
derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i);
-extra = (String[]) ArrayUtils.remove(extra, i);
+if (extra != null)
+extra = (String[]) ArrayUtils.remove(extra, i);
i--;
}
}
+
+if (derivedCols.length == 0)
+return;
-Map toHostMap = derivedToHostMap;
-Map, List> hostToMap = hostToDerivedMap;
+for (int i = 0; i < derivedCols.length; i++) {
+TblColRef derivedCol = derivedCols[i];
+boolean isOneToOne = type == DeriveType.PK_FK ||
ArrayUtils.contains(hostCols, derivedCol) || (extra != null &&
extra[i].contains("1-1"));
+derivedToHostMap.put(derivedCol, new DeriveInfo(type, join,
hostCols, isOneToOne));
+}
Array hostColArray = new Array(hostCols);
-List infoList = hostToMap.get(hostColArray);
+List infoList = hostToDerivedMap.get(hostColArray);
if (infoList == null) {
-hostToMap.put(hostColArray, infoList = new
ArrayList());
+hostToDerivedMap.put(hostColArray, infoList = new
ArrayList());
}
-infoList.add(new DeriveInfo(type, join, derivedCols, false));
-
-for (int i = 0; i < derivedCols.length; i++) {
-TblColRef derivedCol = derivedCols[i];
-boolean isOneToOne = type == DeriveType.PK_FK ||
ArrayUtils.contains(hostCols, derivedCol) || (extra != null &&
extra[i].contains("1-1"));
-toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols,
isOneToOne));
+
+// Merged duplicated derived column
+List whatsLeft = new ArrayList<>();
+for (TblColRef derCol : derivedCols) {
+boolean merged = false;
+for (DeriveInfo existing : infoList) {
+if (existing.type == type &&
existing.join.getPKSide().equals(join.getPKSide())) {
+if (ArrayUtils.contains(existing.columns, derCol)) {
+merged = true;
+break;
+}
+if (type == DeriveType.LOOKUP) {
+existing.columns = (TblColRef[])
ArrayUtils.add(existing.columns, derCol);
+merged = true;
+break;
+
[08/50] [abbrv] kylin git commit: minor code reformat
minor code reformat
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f466a07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f466a07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f466a07
Branch: refs/heads/master-hbase1.x
Commit: 4f466a07bfb5438a1c573699f0b01cd206b0423a
Parents: 0453d3d
Author: Li Yang
Authored: Mon Dec 26 15:59:03 2016 +0800
Committer: Li Yang
Committed: Mon Dec 26 15:59:03 2016 +0800
--
.../java/org/apache/kylin/provision/ProvisionWithoutBuild.java | 2 +-
.../main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f466a07/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
b/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
index 71bb0ca..039d583 100644
---
a/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
+++
b/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
@@ -24,6 +24,6 @@ public class ProvisionWithoutBuild extends
BuildCubeWithEngine {
public static void main(String[] args) throws Exception {
beforeClass();
DeployUtil.deployMetadata();
-
DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty");
+DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f466a07/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index 93f06dd..132e65a 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -163,6 +163,6 @@ public class OLAPSchemaFactory implements SchemaFactory {
}
index++;
}
-out.write(" ]\n");
+out.write("]\n");
}
}
[14/50] [abbrv] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log
introduce CheckUtil to make multiple condition checks clearer in log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2436140e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2436140e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2436140e
Branch: refs/heads/master-hbase1.x
Commit: 2436140ef093c8d5b061124a5dbc2f5a1c3e9f3f
Parents: 182c565
Author: Hongbin Ma
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:15:44 2016 +0800
--
.../org/apache/kylin/common/util/CheckUtil.java | 33
.../apache/kylin/rest/service/QueryService.java | 17 ++
2 files changed, 44 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger =
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message,
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() &&
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled =
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query
cache disabled in BackdoorToggles");
+
if (queryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
}
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() -
startTime);
logger.info("Stats of SQL response: isException: {},
duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()),
String.valueOf(sqlResponse.getDuration()),
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold ||
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() <
kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
+if (checkCondit
[18/50] [abbrv] kylin git commit: add a toggle for dumpping storage partitions
add a toggle for dumpping storage partitions
add more log
temp for dumpped partitions
minor fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bfe8b1a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bfe8b1a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bfe8b1a3
Branch: refs/heads/master-hbase1.x
Commit: bfe8b1a35c6b2c2f377eeb72295622ecb4d71284
Parents: 596ab51
Author: Hongbin Ma
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:17:33 2016 +0800
--
.../kylin/common/debug/BackdoorToggles.java | 30
.../gtrecord/StorageResponseGTScatter.java | 2 ++
2 files changed, 32 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
}
+public static String getPartitionDumpDir() {
+return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+}
+
+public static String getDumpedPartitionDir() {
+return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+}
+
public static int getQueryTimeout() {
String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT =
"DEBUG_TOGGLE_SHARD_ASSIGNMENT";
+/**
+ * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from
storage.
+ * The dumped partitions are used for performance profiling, for example.
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+ }
+ */
+public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR =
"DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+/**
+ * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to
retrieve previously dumped partitions
+ * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+ }
+ */
+public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR =
"DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
// properties on statement may go with this "channel" too
/**
* set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
public Iterator iterator() {
Iterator> shardSubsets =
Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
if (storagePushDownLimit != Integer.MAX_VALUE) {
+logger.info("Using SortedIteratorMergerWithLimit to merge
partitions");
return new SortedIteratorMergerWithLimit(shardSubsets,
storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
} else {
+logger.info("Using Iterators.concat to merge partitions");
return Iterators.concat(shardSubsets);
}
}
[25/50] [abbrv] kylin git commit: minor, invoke toString of translated filter may be expensive
minor, invoke toString of translated filter may be expensive
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7aabe4cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7aabe4cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7aabe4cb
Branch: refs/heads/master-hbase1.x
Commit: 7aabe4cb1d1480f99b4724b20598ba216f44586a
Parents: 9898a07
Author: gaodayue
Authored: Wed Dec 28 12:20:51 2016 +0800
Committer: gaodayue
Committed: Wed Dec 28 12:20:51 2016 +0800
--
.../java/org/apache/kylin/dict/BuiltInFunctionTransformer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7aabe4cb/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index f451c2c..6ef2f96 100755
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -57,13 +57,13 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
//normal case
translated = translateCompareTupleFilter((CompareTupleFilter)
tupleFilter);
if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {"
+ translated + "}");
+logger.debug("Translated {{}} to IN clause: {{}}",
tupleFilter, translated);
}
} else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
//like case
translated =
translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {"
+ translated + "}");
+logger.debug("Translated {{}} to IN clause: {{}}",
tupleFilter, translated);
}
} else if (tupleFilter instanceof LogicalTupleFilter) {
@SuppressWarnings("unchecked")
[30/50] [abbrv] kylin git commit: minor,cube measures add dimensions
minor,cube measures add dimensions
Signed-off-by: zhongjian
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a7c25d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a7c25d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a7c25d
Branch: refs/heads/master-hbase1.x
Commit: 73a7c25dfeb28764658af3e73065aa04cc1b92cd
Parents: 76e321b
Author: chenzhx <346839...@qq.com>
Authored: Wed Dec 28 16:18:01 2016 +0800
Committer: zhongjian
Committed: Wed Dec 28 16:31:30 2016 +0800
--
webapp/app/js/controllers/cubeMeasures.js | 11 ++-
webapp/app/partials/cubeDesigner/measures.html | 6 --
2 files changed, 14 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a7c25d/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js
b/webapp/app/js/controllers/cubeMeasures.js
index 15c330a..c8548ce 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -84,6 +84,15 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope,
$modal,MetaModel,cubes
if(!!measure && measure.function.parameter.next_parameter){
$scope.nextPara.value = measure.function.parameter.next_parameter.value;
}
+if($scope.newMeasure.function.parameter.value){
+
if($scope.metaModel.model.metrics&&$scope.metaModel.model.metrics.indexOf($scope.newMeasure.function.parameter.value)!=-1){
+ $scope.newMeasure.showDim=false;
+ }else{
+ $scope.newMeasure.showDim=true;
+ }
+}else{
+ $scope.newMeasure.showDim=false;
+}
if($scope.newMeasure.function.expression=="TOP_N"){
$scope.convertedColumns=[];
if($scope.newMeasure.function.configuration==null){
@@ -353,7 +362,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope,
$modal,MetaModel,cubes
if($scope.newMeasure.function.parameter.type=="column"&&$scope.newMeasure.function.expression!=="COUNT_DISTINCT"){
var column = $scope.newMeasure.function.parameter.value;
- if(column){
+ if(column&&(typeof column=="string")){
var colType = $scope.getColumnType(VdmUtil.removeNameSpace(column),
VdmUtil.getNameSpaceAliasName(column)); // $scope.getColumnType defined in
cubeEdit.js
}
if(colType==""||!colType){
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a7c25d/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html
b/webapp/app/partials/cubeDesigner/measures.html
index 9eb04c3..4d2d8ab 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -150,14 +150,15 @@
-- Select a Column
--
+
@@ -171,6 +172,7 @@
-- Select a Column
--
+ Also Show
Dimensions
[16/50] [abbrv] kylin git commit: minor code refactors
minor code refactors
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa4a5ee2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa4a5ee2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa4a5ee2
Branch: refs/heads/master-hbase1.x
Commit: fa4a5ee269eea66eee5b8ab8278305dfe0144c92
Parents: 00de441
Author: Hongbin Ma
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:15:44 2016 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 6 ++-
.../org/apache/kylin/common/util/BasicTest.java | 1 +
.../gtrecord/DummyPartitionStreamer.java| 40
.../storage/gtrecord/IPartitionStreamer.java| 26 +
.../gtrecord/StorageResponseGTScatter.java | 9 +++--
.../apache/kylin/rest/service/QueryService.java | 6 ++-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 9 +++--
7 files changed, 87 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8080577..d73b694 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -750,7 +750,7 @@ abstract public class KylinConfigBase implements
Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments",
"true"));
}
-@Deprecated//Limit is good even it's large. This config is meaning less
since we already have scan threshold
+@Deprecated //Limit is good even it's large. This config is meaning less
since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown",
"1"));
}
@@ -759,6 +759,10 @@ abstract public class KylinConfigBase implements
Serializable {
return Integer.parseInt(getOptional("kylin.query.scan-threshold",
"1000"));
}
+public int getLargeQueryThreshold() {
+return
Integer.parseInt(getOptional("kylin.query.large-query-threshold",
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
public int getDerivedInThreshold() {
return
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
"20"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import com.google.common.collect.Iterators;
import org.apache.commons.lang3.time.FastDateFormat;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations un
[10/50] [abbrv] kylin git commit: minor, fix UT
minor, fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9a8402d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9a8402d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9a8402d
Branch: refs/heads/master-hbase1.x
Commit: f9a8402d57755a9c3c3132ff676cf3c7492775c9
Parents: a78b650
Author: Billy Liu
Authored: Tue Dec 27 00:02:35 2016 +0800
Committer: Billy Liu
Committed: Tue Dec 27 00:02:35 2016 +0800
--
.../kylin/dict/DictionaryProviderTest.java | 36 +---
1 file changed, 32 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f9a8402d/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
--
diff --git
a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
index a4aee76..4b386a7 100644
---
a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
+++
b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import static org.junit.Assert.assertTrue;
@@ -13,13 +31,23 @@ import java.util.Iterator;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-/**
- * Created by xiefan on 16-11-23.
- */
-public class DictionaryProviderTest {
+public class DictionaryProviderTest extends LocalFileMetadataTestCase{
+
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
@Test
public void testReadWrite() throws Exception {
[24/50] [abbrv] kylin git commit: KYLIN-2318 bug fix again...
KYLIN-2318 bug fix again...
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9898a07b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9898a07b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9898a07b
Branch: refs/heads/master-hbase1.x
Commit: 9898a07b1451c29b25fb295b22ed62015073df93
Parents: cccf3a9
Author: Hongbin Ma
Authored: Wed Dec 28 10:20:31 2016 +0800
Committer: Hongbin Ma
Committed: Wed Dec 28 10:21:28 2016 +0800
--
.../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9898a07b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
-BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+if (sqlRequest.getBackdoorToggles() != null)
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
[26/50] [abbrv] kylin git commit: minor set session expire time to 8 hours by deault
minor set session expire time to 8 hours by deault Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8747e24d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8747e24d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8747e24d Branch: refs/heads/master-hbase1.x Commit: 8747e24da5053cc8bda8ec27b48c01bd97b7c63b Parents: 7aabe4c Author: zhongjian Authored: Wed Dec 28 14:55:50 2016 +0800 Committer: zhongjian Committed: Wed Dec 28 14:55:50 2016 +0800 -- server/src/main/webapp/WEB-INF/web.xml | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/8747e24d/server/src/main/webapp/WEB-INF/web.xml -- diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml index 8a43b33..b9cf620 100644 --- a/server/src/main/webapp/WEB-INF/web.xml +++ b/server/src/main/webapp/WEB-INF/web.xml @@ -30,6 +30,10 @@ /index.html + +480 + + log4jConfigLocation classpath:kylin-server-log4j.properties
[03/50] [abbrv] kylin git commit: KYLIN-2314 Use col identity instead of col name in dictionary refs
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv -- diff --git a/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv b/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv new file mode 100644 index 000..aced0e3 --- /dev/null +++ b/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv @@ -0,0 +1,244 @@ +AD,42.546245,1.601554,Andorra +AE,23.424076,53.847818,United Arab Emirates +AF,33.93911,67.709953,Afghanistan +AG,17.060816,-61.796428,Antigua and Barbuda +AI,18.220554,-63.068615,Anguilla +AL,41.153332,20.168331,Albania +AM,40.069099,45.038189,Armenia +AN,12.226079,-69.060087,Netherlands Antilles +AO,-11.202692,17.873887,Angola +AQ,-75.250973,-0.071389,Antarctica +AR,-38.416097,-63.616672,Argentina +AS,-14.270972,-170.132217,American Samoa +AT,47.516231,14.550072,Austria +AU,-25.274398,133.775136,Australia +AW,12.52111,-69.968338,Aruba +AZ,40.143105,47.576927,Azerbaijan +BA,43.915886,17.679076,Bosnia and Herzegovina +BB,13.193887,-59.543198,Barbados +BD,23.684994,90.356331,Bangladesh +BE,50.503887,4.469936,Belgium +BF,12.238333,-1.561593,Burkina Faso +BG,42.733883,25.48583,Bulgaria +BH,25.930414,50.637772,Bahrain +BI,-3.373056,29.918886,Burundi +BJ,9.30769,2.315834,Benin +BM,32.321384,-64.75737,Bermuda +BN,4.535277,114.727669,Brunei +BO,-16.290154,-63.588653,Bolivia +BR,-14.235004,-51.92528,Brazil +BS,25.03428,-77.39628,Bahamas +BT,27.514162,90.433601,Bhutan +BV,-54.423199,3.413194,Bouvet Island +BW,-22.328474,24.684866,Botswana +BY,53.709807,27.953389,Belarus +BZ,17.189877,-88.49765,Belize +CA,56.130366,-106.346771,Canada +CC,-12.164165,96.870956,Cocos [Keeling] Islands +CD,-4.038333,21.758664,Congo [DRC] +CF,6.61,20.939444,Central African Republic +CG,-0.228021,15.827659,Congo [Republic] +CH,46.818188,8.227512,Switzerland +CI,7.539989,-5.54708,C?te d'Ivoire +CK,-21.236736,-159.777671,Cook Islands +CL,-35.675147,-71.542969,Chile +CM,7.369722,12.354722,Cameroon +CN,35.86166,104.195397,China +CO,4.570868,-74.297333,Colombia +CR,9.748917,-83.753428,Costa Rica +CU,21.521757,-77.781167,Cuba +CV,16.002082,-24.013197,Cape Verde +CX,-10.447525,105.690449,Christmas Island +CY,35.126413,33.429859,Cyprus +CZ,49.817492,15.472962,Czech Republic +DE,51.165691,10.451526,Germany +DJ,11.825138,42.590275,Djibouti +DK,56.26392,9.501785,Denmark +DM,15.414999,-61.370976,Dominica +DO,18.735693,-70.162651,Dominican Republic +DZ,28.033886,1.659626,Algeria +EC,-1.831239,-78.183406,Ecuador +EE,58.595272,25.013607,Estonia +EG,26.820553,30.802498,Egypt +EH,24.215527,-12.885834,Western Sahara +ER,15.179384,39.782334,Eritrea +ES,40.463667,-3.74922,Spain +ET,9.145,40.489673,Ethiopia +FI,61.92411,25.748151,Finland +FJ,-16.578193,179.414413,Fiji +FK,-51.796253,-59.523613,Falkland Islands [Islas Malvinas] +FM,7.425554,150.550812,Micronesia +FO,61.892635,-6.911806,Faroe Islands +FR,46.227638,2.213749,France +GA,-0.803689,11.609444,Gabon +GB,55.378051,-3.435973,United Kingdom +GD,12.262776,-61.604171,Grenada +GE,42.315407,43.356892,Georgia +GF,3.933889,-53.125782,French Guiana +GG,49.465691,-2.585278,Guernsey +GH,7.946527,-1.023194,Ghana +GI,36.137741,-5.345374,Gibraltar +GL,71.706936,-42.604303,Greenland +GM,13.443182,-15.310139,Gambia +GN,9.945587,-9.696645,Guinea +GP,16.995971,-62.067641,Guadeloupe +GQ,1.650801,10.267895,Equatorial Guinea +GR,39.074208,21.824312,Greece +GS,-54.429579,-36.587909,South Georgia and the South Sandwich Islands +GT,15.783471,-90.230759,Guatemala +GU,13.444304,144.793731,Guam +GW,11.803749,-15.180413,Guinea-Bissau +GY,4.860416,-58.93018,Guyana +GZ,31.354676,34.308825,Gaza Strip +HK,22.396428,114.109497,Hong Kong +HM,-53.08181,73.504158,Heard Island and McDonald Islands +HN,15.19,-86.241905,Honduras +HR,45.1,15.2,Croatia +HT,18.971187,-72.285215,Haiti +HU,47.162494,19.503304,Hungary +ID,-0.789275,113.921327,Indonesia +IE,53.41291,-8.24389,Ireland +IL,31.046051,34.851612,Israel +IM,54.236107,-4.548056,Isle of Man +IN,20.593684,78.96288,India +IO,-6.343194,71.876519,British Indian Ocean Territory +IQ,33.223191,43.679291,Iraq +IR,32.427908,53.688046,Iran +IS,64.963051,-19.020835,Iceland +IT,41.87194,12.56738,Italy +JE,49.214439,-2.13125,Jersey +JM,18.109581,-77.297508,Jamaica +JO,30.585164,36.238414,Jordan +JP,36.204824,138.252924,Japan +KE,-0.023559,37.906193,Kenya +KG,41.20438,74.766098,Kyrgyzstan +KH,12.565679,104.990963,Cambodia +KI,-3.370417,-168.734039,Kiribati +KM,-11.875001,43.872219,Comoros +KN,17.357822,-62.782998,Saint Kitts and Nevis +KP,40.339852,127.510093,North Korea +KR,35.907757,127.766922,South Korea +KW,29.31166,47.481766,Kuwait +KY,19.513469,-80.566956,Cayman Islands +KZ,48.019573,66.923684,Kazakhstan +LA,19.85627,102.495496,Laos +LB,33.854721,35.862285,Lebanon +LC,13.909444,-60.978893,Saint Lucia +LI,47.166,9.555373,Liechtenstein +LK,7.873054,80.771797,Sri Lanka +LR,6.428055,-9.429499,Liberia +LS,-29.609988,28
[22/50] [abbrv] kylin git commit: KYLIN-2327 fix UT
KYLIN-2327 fix UT Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/84ec908b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/84ec908b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/84ec908b Branch: refs/heads/master-hbase1.x Commit: 84ec908b57cff0d3167cc35a9e4ea6d4b7bf94ce Parents: 3e0edc8 Author: Billy Liu Authored: Wed Dec 28 08:42:33 2016 +0800 Committer: Billy Liu Committed: Wed Dec 28 08:42:33 2016 +0800 -- .../src/test/java/org/apache/kylin/common/util/BasicTest.java | 1 - kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java| 1 - 2 files changed, 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/84ec908b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java -- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index bbaab09..5512147b 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -34,7 +34,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.google.common.collect.Iterators; import org.apache.commons.lang3.time.FastDateFormat; import org.junit.Ignore; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/kylin/blob/84ec908b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 3ae7521..4f1f591 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -48,7 +48,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; import org.apache.kylin.query.schema.OLAPSchemaFactory;
[07/50] [abbrv] kylin git commit: KYLIN-2301 various fixes to support NOT IN subquery
KYLIN-2301 various fixes to support NOT IN subquery
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0453d3da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0453d3da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0453d3da
Branch: refs/heads/master-hbase1.x
Commit: 0453d3daada2e32f79954dfd12ac9f19062e825c
Parents: ed8b168
Author: Li Yang
Authored: Fri Dec 23 08:17:28 2016 +0800
Committer: Li Yang
Committed: Mon Dec 26 15:26:25 2016 +0800
--
.../kylin/metadata/filter/LogicalTupleFilter.java | 13 -
.../org/apache/kylin/metadata/filter/TupleFilter.java | 7 ++-
.../org/apache/kylin/metadata/model/TblColRef.java | 7 ++-
.../java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../src/test/resources/query/sql_subquery/query13.sql | 7 +++
.../apache/kylin/query/relnode/OLAPAggregateRel.java | 11 +--
.../org/apache/kylin/query/relnode/OLAPJoinRel.java| 8 +++-
7 files changed, 32 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 1744309..373acdb 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -47,19 +47,6 @@ public class LogicalTupleFilter extends TupleFilter {
return cloneTuple;
}
-//private TupleFilter reverseNestedNots(TupleFilter filter, int depth)
{
-//if ((filter instanceof LogicalTupleFilter) && (filter.operator
== FilterOperatorEnum.NOT)) {
-//assert (filter.children.size() == 1);
-//return reverseNestedNots(filter.children.get(0), depth + 1);
-//}
-//
-//if (depth % 2 == 1) {
-//return filter;
-//} else {
-//return filter.reverse();
-//}
-//}
-
@Override
public TupleFilter reverse() {
switch (operator) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 285172c..31de50b 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -27,6 +27,8 @@ import java.util.Set;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
@@ -37,6 +39,8 @@ import com.google.common.collect.Maps;
*/
public abstract class TupleFilter {
+static final Logger logger = LoggerFactory.getLogger(TupleFilter.class);
+
public enum FilterOperatorEnum {
EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8),
IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31),
DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), EVAL_FUNC(37),
UNSUPPORTED(38);
@@ -112,7 +116,8 @@ public abstract class TupleFilter {
}
public TupleFilter reverse() {
-throw new UnsupportedOperationException();
+logger.warn("Cannot reverse " + this + ", loosen the filter to true");
+return ConstantTupleFilter.TRUE;
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index fd0224f..88abb9d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -27,7 +27,7 @@ import org.apache.kylin.metadata.datatype.DataType;
/**
*/
-@SuppressWarnings("serial")
+@SuppressWarnings({ "serial", "deprecation" })
public class TblColRef implements Serializable {
private static final String INNER_TABLE_NAME = "_kylin_table";
@@ -82
[12/50] [abbrv] kylin git commit: minor, drop scan threshold in KylinTestBase
minor, drop scan threshold in KylinTestBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00de4411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00de4411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00de4411
Branch: refs/heads/master-hbase1.x
Commit: 00de44116126bd585f906c25606579a1ce2961f3
Parents: 74c2d6b
Author: Li Yang
Authored: Tue Dec 27 18:38:16 2016 +0800
Committer: Li Yang
Committed: Tue Dec 27 18:38:16 2016 +0800
--
kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java | 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/00de4411/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 0a795ef..3ae7521 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -631,7 +631,6 @@ public class KylinTestBase {
//setup cube conn
File olapTmp =
OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME,
config);
Properties props = new Properties();
-props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "20001");
cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" +
olapTmp.getAbsolutePath(), props);
//setup h2
[01/50] [abbrv] kylin git commit: KYLIN-2306 fall back to abstract when NoClassFoundException [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-hbase1.x 9690649b8 -> 982e8cf2a (forced update)
KYLIN-2306 fall back to abstract when NoClassFoundException
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a40296
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a40296
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a40296
Branch: refs/heads/master-hbase1.x
Commit: 73a40296e33921a000cb97faa3321dd53b8dbf41
Parents: 488283d
Author: Billy Liu
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu
Committed: Fri Dec 23 23:49:20 2016 +0800
--
.../kylin/job/execution/ExecutableManager.java | 59
1 file changed, 35 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a40296/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
private static final Logger logger =
LoggerFactory.getLogger(ExecutableManager.class);
private static final ConcurrentHashMap
CACHE = new ConcurrentHashMap();
-
+
private final KylinConfig config;
private final ExecutableDao executableDao;
+private ExecutableManager(KylinConfig config) {
+logger.info("Using metadata url: " + config);
+this.config = config;
+this.executableDao = ExecutableDao.getInstance(config);
+}
+
public static ExecutableManager getInstance(KylinConfig config) {
ExecutableManager r = CACHE.get(config);
if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
CACHE.clear();
}
-private ExecutableManager(KylinConfig config) {
-logger.info("Using metadata url: " + config);
-this.config = config;
-this.executableDao = ExecutableDao.getInstance(config);
+private static ExecutablePO parse(AbstractExecutable executable) {
+ExecutablePO result = new ExecutablePO();
+result.setName(executable.getName());
+result.setUuid(executable.getId());
+result.setType(executable.getClass().getName());
+result.setParams(executable.getParams());
+if (executable instanceof ChainedExecutable) {
+List tasks = Lists.newArrayList();
+for (AbstractExecutable task : ((ChainedExecutable)
executable).getTasks()) {
+tasks.add(parse(task));
+}
+result.setTasks(tasks);
+}
+return result;
}
public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
}
-
public void rollbackJob(String jobId, String stepId) {
AbstractExecutable job = getJob(jobId);
if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
addJobInfo(id, info);
}
-private static ExecutablePO parse(AbstractExecutable executable) {
-ExecutablePO result = new ExecutablePO();
-result.setName(executable.getName());
-result.setUuid(executable.getId());
-result.setType(executable.getClass().getName());
-result.setParams(executable.getParams());
-if (executable instanceof ChainedExecutable) {
-List tasks = Lists.newArrayList();
-for (AbstractExecutable task : ((ChainedExecutable)
executable).getTasks()) {
-tasks.add(parse(task));
-}
-result.setTasks(tasks);
-}
-return result;
-}
-
private AbstractExecutable parseTo(ExecutablePO executablePO) {
if (executablePO == null) {
logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
logger.warn("executablePO is null");
return null;
}
+String type = executablePO.getType();
try {
-Class clazz =
ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+Class clazz = null;
+try {
+clazz = ClassUtil.forName(type, AbstractExecutable.class);
+} catch (ClassNotFoundException e) {
+clazz = ClassUtil.forName(expectedClass.getName(),
AbstractExecutable.class);
+}
Constructor constructor =
cla
[40/50] [abbrv] kylin git commit: KYLIN-2323 Refactor table load/unload
KYLIN-2323 Refactor table load/unload
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2e2a81c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2e2a81c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2e2a81c
Branch: refs/heads/master-hbase1.x
Commit: e2e2a81c1f1d7e3f9af5c81bb4f1ad14d2d1b859
Parents: 6ea03b8
Author: Billy Liu
Authored: Thu Dec 29 18:48:38 2016 +0800
Committer: Billy Liu
Committed: Thu Dec 29 18:48:38 2016 +0800
--
.../hive/ITHiveSourceTableLoaderTest.java | 2 +-
.../rest/controller/StreamingController.java| 22 +-
.../kylin/rest/controller/TableController.java | 268
.../apache/kylin/rest/service/CubeService.java | 96 --
.../apache/kylin/rest/service/TableService.java | 318 +++
.../source/hive/HiveSourceTableLoader.java | 2 +-
webapp/app/js/controllers/sourceMeta.js | 16 +-
webapp/app/js/services/tables.js| 1 -
8 files changed, 391 insertions(+), 334 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
index c4f0777..7aff3ba 100644
---
a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++
b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
@@ -45,7 +45,7 @@ public class ITHiveSourceTableLoaderTest extends
HBaseMetadataTestCase {
public void test() throws IOException {
KylinConfig config = getTestConfig();
String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT",
"EDW.TEST_CAL_DT" };
-Set loaded = HiveSourceTableLoader.reloadHiveTables(toLoad,
config);
+Set loaded = HiveSourceTableLoader.loadHiveTables(toLoad,
config);
assertTrue(loaded.size() == toLoad.length);
for (String str : toLoad)
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e2a81c/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index e04ebc8..0ced9ad 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -23,10 +23,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -34,9 +32,9 @@ import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.StreamingRequest;
-import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.KafkaConfigService;
import org.apache.kylin.rest.service.StreamingService;
+import org.apache.kylin.rest.service.TableService;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +67,7 @@ public class StreamingController extends BasicController {
@Autowired
private KafkaConfigService kafkaConfigService;
@Autowired
-private CubeService cubeMgmtService;
+private TableService tableService;
@RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
@ResponseBody
@@ -113,10 +111,7 @@ public class StreamingController extends BasicController {
boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
try {
-tableDesc.setUuid(UUID.randomUUID().toString());
-MetadataManager metaMgr =
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-metaMgr.saveSourceTable(tableDesc);
-cubeMgmtService.syncTableToProject(new String[] {
tableDesc.getIdentity() }, project);
+tableService.addStreamingTable(tableDesc, project);
} catch (IOException e) {
throw new BadRequestExce
[50/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/982e8cf2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/982e8cf2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/982e8cf2 Branch: refs/heads/master-hbase1.x Commit: 982e8cf2a9d406fd35009f7c1a857dc9db8c032d Parents: fdcb9b8 Author: shaofengshi Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: lidongsjtu Committed: Fri Dec 30 16:08:27 2016 +0800 -- examples/test_case_data/sandbox/hbase-site.xml | 19 +--- .../kylin/provision/BuildCubeWithEngine.java| 12 +-- pom.xml | 12 +-- .../kylin/rest/security/AclHBaseStorage.java| 4 +- .../rest/security/MockAclHBaseStorage.java | 8 +- .../apache/kylin/rest/security/MockHTable.java | 95 .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/AclService.java | 25 +++--- .../apache/kylin/rest/service/CubeService.java | 35 +++- .../apache/kylin/rest/service/QueryService.java | 24 +++-- .../apache/kylin/rest/service/UserService.java | 17 ++-- .../kylin/storage/hbase/HBaseConnection.java| 44 - .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++ .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++--- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +- .../coprocessor/endpoint/CubeVisitService.java | 4 +- .../storage/hbase/steps/CubeHTableUtil.java | 16 ++-- .../storage/hbase/steps/DeprecatedGCStep.java | 23 ++--- .../storage/hbase/steps/HBaseCuboidWriter.java | 7 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--- .../storage/hbase/util/CleanHtableCLI.java | 12 +-- .../storage/hbase/util/CubeMigrationCLI.java| 36 .../hbase/util/CubeMigrationCheckCLI.java | 17 ++-- .../hbase/util/DeployCoprocessorCLI.java| 27 +++--- .../hbase/util/ExtendCubeToHybridCLI.java | 8 +- .../hbase/util/GridTableHBaseBenchmark.java | 34 +++ .../kylin/storage/hbase/util/HBaseClean.java| 18 ++-- .../hbase/util/HBaseRegionSizeCalculator.java | 35 .../kylin/storage/hbase/util/HBaseUsage.java| 9 +- .../storage/hbase/util/HbaseStreamingInput.java | 30 +++ .../hbase/util/HtableAlterMetadataCLI.java | 9 +- .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++-- .../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++-- .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-- .../storage/hbase/util/StorageCleanupJob.java | 20 +++-- .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++-- .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++-- .../kylin/tool/ExtendCubeToHybridCLI.java | 8 +- 39 files changed, 363 insertions(+), 431 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/982e8cf2/examples/test_case_data/sandbox/hbase-site.xml -- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 46d5345..734908e 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -190,22 +190,5 @@ zookeeper.znode.parent /hbase-unsecure - -hbase.client.pause -100 -General client pause value. Used mostly as value to wait -before running a retry of a failed get, region lookup, etc. -See hbase.client.retries.number for description of how we backoff from -this initial pause amount and how this pause works w/ retries. - - -hbase.client.retries.number -5 -Maximum retries. Used as maximum for all retryable -operations such as the getting of a cell's value, starting a row update, -etc. Retry interval is a rough function based on hbase.client.pause. At -first we retry at this interval but then with backoff, we pretty quickly reach -retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup -ramps up. Change this setting and hbase.client.pause to suit your workload. - + http://git-wip-us.apache.org/repos/asf/kylin/blob/982e8cf2/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java -- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 1c7c64d..1c4b6b2 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/k
[39/50] [abbrv] kylin git commit: KYLIN-2328 Reduce the size of metadata uploaded to distributed cache
KYLIN-2328 Reduce the size of metadata uploaded to distributed cache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ea03b86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ea03b86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ea03b86
Branch: refs/heads/master-hbase1.x
Commit: 6ea03b8638d72853959aba0666aea18f1ba97391
Parents: 64c3c61
Author: gaodayue
Authored: Wed Dec 28 15:27:49 2016 +0800
Committer: gaodayue
Committed: Thu Dec 29 18:39:13 2016 +0800
--
.../engine/mr/common/AbstractHadoopJob.java | 43 ++--
.../apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
.../engine/mr/steps/FactDistinctColumnsJob.java | 7 ++--
.../kylin/engine/mr/steps/InMemCuboidJob.java | 10 ++---
.../kylin/engine/mr/steps/MergeCuboidJob.java | 3 +-
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
.../source/kafka/hadoop/KafkaFlatTableJob.java | 3 --
.../kylin/storage/hbase/steps/CubeHFileJob.java | 11 +++--
8 files changed, 48 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ea03b86/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 80636d3..4693ac3 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -29,7 +29,6 @@ import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -449,33 +448,49 @@ public abstract class AbstractHadoopJob extends
Configured implements Tool {
}
}
-protected void attachKylinPropsAndMetadata(TableDesc table, Configuration
conf) throws IOException {
+protected void attachTableMetadata(TableDesc table, Configuration conf)
throws IOException {
Set dumpList = new LinkedHashSet<>();
dumpList.add(table.getResourcePath());
attachKylinPropsAndMetadata(dumpList,
KylinConfig.getInstanceFromEnv(), conf);
}
-protected void attachKylinPropsAndMetadata(CubeInstance cube,
Configuration conf) throws IOException {
-// write cube / model_desc / cube_desc / dict / table
+protected void attachCubeMetadata(CubeInstance cube, Configuration conf)
throws IOException {
+attachKylinPropsAndMetadata(collectCubeMetadata(cube),
cube.getConfig(), conf);
+}
+
+protected void attachCubeMetadataWithDict(CubeInstance cube, Configuration
conf) throws IOException {
+Set dumpList = new LinkedHashSet<>();
+dumpList.addAll(collectCubeMetadata(cube));
+for (CubeSegment segment : cube.getSegments()) {
+dumpList.addAll(segment.getDictionaryPaths());
+}
+attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+}
+
+protected void attachSegmentMetadataWithDict(CubeSegment segment,
Configuration conf) throws IOException {
+Set dumpList = new LinkedHashSet<>();
+dumpList.addAll(collectCubeMetadata(segment.getCubeInstance()));
+dumpList.addAll(segment.getDictionaryPaths());
+attachKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
+}
+
+private Set collectCubeMetadata(CubeInstance cube) {
+// cube, model_desc, cube_desc, table
Set dumpList = new LinkedHashSet<>();
dumpList.add(cube.getResourcePath());
dumpList.add(cube.getDescriptor().getModel().getResourcePath());
dumpList.add(cube.getDescriptor().getResourcePath());
-for (TableRef tableRef:
cube.getDescriptor().getModel().getAllTables()) {
+for (TableRef tableRef :
cube.getDescriptor().getModel().getAllTables()) {
TableDesc table = tableRef.getTableDesc();
dumpList.add(table.getResourcePath());
-List dependentResources =
SourceFactory.getMRDependentResources(table);
-dumpList.addAll(dependentResources);
-}
-for (CubeSegment segment : cube.getSegments()) {
-dumpList.addAll(segment.getDictionaryPaths());
+dumpList.addAll(SourceFactory.getMRDependentResources(table));
}
-attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+return dumpList;
}
-protected void attachKylinPropsAndMetadata(Set dumpList,
KylinConfig kylinConfig, Configuration conf) throws IOException {
+private void attachKylinPropsAndMetadata(Set dumpLi
[45/50] [abbrv] kylin git commit: minor, fix NPE and check table using by model code in MetadataManager
minor, fix NPE and check table using by model code in MetadataManager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32d72444
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32d72444
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32d72444
Branch: refs/heads/master-hbase1.x
Commit: 32d724440657fd97462d4d10f9a5d91ba0d5a6da
Parents: a9ee6d3
Author: Billy Liu
Authored: Fri Dec 30 11:06:14 2016 +0800
Committer: Billy Liu
Committed: Fri Dec 30 11:06:30 2016 +0800
--
.../apache/kylin/metadata/MetadataManager.java | 21 ++--
1 file changed, 11 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/32d72444/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index d0e4b42..7290d4c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -39,6 +39,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
@@ -395,6 +396,7 @@ public class MetadataManager {
// remove old json
if (name == null) {
getStore().deleteResource(path);
+return null;
}
srcTableExdMap.putLocal(name, t);
@@ -495,19 +497,16 @@ public class MetadataManager {
}
public boolean isTableInModel(String tableName, String projectName) throws
IOException {
-for (DataModelDesc modelDesc : getModels(projectName)) {
-if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
-return true;
-}
-}
-return false;
+return getModelsUsingTable(tableName, projectName).size() > 0;
}
public List getModelsUsingTable(String tableName, String
projectName) throws IOException {
List models = new ArrayList<>();
for (DataModelDesc modelDesc : getModels(projectName)) {
-if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
-models.add(modelDesc.getName());
+for(TableRef tableRef : modelDesc.getAllTables()){
+if(tableRef.getTableName().equalsIgnoreCase(tableName)){
+models.add(modelDesc.getName());
+}
}
}
return models;
@@ -515,8 +514,10 @@ public class MetadataManager {
public boolean isTableInAnyModel(String tableName) {
for (DataModelDesc modelDesc : getModels()) {
-if (modelDesc.getAllTables().contains(tableName.toUpperCase())) {
-return true;
+for(TableRef tableRef : modelDesc.getAllTables()){
+if(tableRef.getTableName().equalsIgnoreCase(tableName)){
+return true;
+}
}
}
return false;
[42/50] [abbrv] kylin git commit: minor, fix wrong prop name in kylin.properties
minor, fix wrong prop name in kylin.properties Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75c8d13c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75c8d13c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75c8d13c Branch: refs/heads/master-hbase1.x Commit: 75c8d13c708e66c995cdea494f8656b72636a4d6 Parents: 020cf5f Author: lidongsjtu Authored: Thu Dec 29 22:53:45 2016 +0800 Committer: lidongsjtu Committed: Thu Dec 29 22:57:54 2016 +0800 -- build/conf/kylin.properties | 2 +- examples/test_case_data/sandbox/kylin.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/75c8d13c/build/conf/kylin.properties -- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 131a725..5e7d235 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -30,7 +30,7 @@ kylin.env=QA ### SERVER | WEB ### # Kylin server mode, valid value [all, query, job] -kyin.server.mode=all +kylin.server.mode=all # List of web servers in use, this enables one web server instance to sync up with other servers. kylin.server.cluster-servers=localhost:7070 http://git-wip-us.apache.org/repos/asf/kylin/blob/75c8d13c/examples/test_case_data/sandbox/kylin.properties -- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 20bc427..1724619 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -18,7 +18,7 @@ ### SERVICE ### # Kylin server mode, valid value [all, query, job] -kyin.server.mode=all +kylin.server.mode=all # Optional information for the owner of kylin platform, it can be your team's email # Currently it will be attached to each kylin's htable attribute
[20/50] [abbrv] kylin git commit: KYLIN-2327 enable check-style for test code
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
--
diff --git
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
deleted file mode 100644
index 2edf11e..000
---
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-//*/
-//
-//package org.apache.kylin.storage.minicluster;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.sql.Connection;
-//import java.sql.DriverManager;
-//import java.sql.SQLException;
-//
-//import org.apache.commons.io.FileUtils;
-//import org.apache.hadoop.hdfs.MiniDFSCluster;
-//import org.apache.hadoop.hive.conf.HiveConf;
-//import org.apache.hadoop.hive.service.HiveInterface;
-//import org.apache.hadoop.mapred.JobConf;
-//import org.apache.hadoop.mapred.MiniMRCluster;
-//
-///**
-// * This is a test case to verify whether the query can be executed on Hive
minicluster;
-// * You need set $HADOOP_HOME environment parameter before run it;
-// * @author shaoshi
-// *
-// */
-//public class HiveMiniClusterTest extends HiveJDBCClientTest {
-//public static final File HIVE_BASE_DIR = new File("target/hive");
-//public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR +
"/scratchdir");
-//public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR
+ "/localscratchdir");
-//public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR +
"/metastoredb");
-//public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR +
"/logs");
-//public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp");
-//public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR +
"/warehouse");
-//public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR +
"/testdata");
-//public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR +
"/hadooptmp");
-//protected HiveInterface client;
-//
-//protected MiniDFSCluster miniDFS;
-//protected MiniMRCluster miniMR;
-//
-////@Before
-//public void setup() {
-//super.setup();
-//startHiveMiniCluster();
-//}
-//
-//protected void startHiveMiniCluster() {
-////Create and configure location for hive to dump junk in target
folder
-//try {
-//FileUtils.forceMkdir(HIVE_BASE_DIR);
-//FileUtils.forceMkdir(HIVE_SCRATCH_DIR);
-//FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR);
-//FileUtils.forceMkdir(HIVE_LOGS_DIR);
-//FileUtils.forceMkdir(HIVE_TMP_DIR);
-//FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR);
-//FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR);
-//FileUtils.forceMkdir(HIVE_TESTDATA_DIR);
-//} catch (IOException e1) {
-//e1.printStackTrace();
-//System.exit(1);
-//}
-//
-//System.setProperty("javax.jdo.option.ConnectionURL",
"jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() +
";create=true");
-//System.setProperty("hive.metastore.warehouse.dir",
HIVE_WAREHOUSE_DIR.getAbsolutePath());
-//System.setProperty("hive.exec.scratchdir",
HIVE_SCRATCH_DIR.getAbsolutePath());
-//System.setProperty("hive.exec.local.scratchdir",
HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath());
-//System.setProperty("hive.metastore.metadb.dir",
HIVE_METADB_DIR.getAbsolutePath());
-//System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath());
-//System.setProperty("hive.querylog.location",
HIVE_TMP_DIR.getAbsolutePath());
-//System.setProperty("hadoop.tmp.dir",
HIVE_HADOOP_TMP_DIR.getAbsolutePath());
-//System.setProperty("derby.stream.error.file",
HIVE_BASE_DIR.getAbsolutePath() + "/derby.log");
-//
-//// custom properties
-//System.setProperty("hive.server2.long.poll
[41/50] [abbrv] kylin git commit: minor improvement on logging for shard/reducer sizing
minor improvement on logging for shard/reducer sizing
temp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/020cf5f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/020cf5f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/020cf5f0
Branch: refs/heads/master-hbase1.x
Commit: 020cf5f00ebeb3514493652629e408906ac1786e
Parents: e2e2a81
Author: Hongbin Ma
Authored: Thu Dec 29 14:13:36 2016 +0800
Committer: Hongbin Ma
Committed: Thu Dec 29 18:51:50 2016 +0800
--
.../apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
.../kylin/engine/mr/steps/CuboidReducer.java| 2 +-
.../engine/mr/steps/InMemCuboidReducer.java | 2 +-
.../engine/mr/steps/LayerReducerNumSizing.java | 84
.../engine/mr/steps/LayerReduerNumSizing.java | 82 ---
.../kylin/engine/mr/steps/MergeCuboidJob.java | 2 +-
6 files changed, 88 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index bd305c1..ef25b55 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -131,7 +131,7 @@ public class CuboidJob extends AbstractHadoopJob {
// add metadata to distributed cache
attachSegmentMetadataWithDict(segment, job.getConfiguration());
-LayerReduerNumSizing.setReduceTaskNum(job, segment,
getTotalMapInputMB(), nCuboidLevel);
+LayerReducerNumSizing.setReduceTaskNum(job, segment,
getTotalMapInputMB(), nCuboidLevel);
this.deletePath(job.getConfiguration(), output);
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index b1d4aaa..afd29e3 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -91,7 +91,7 @@ public class CuboidReducer extends KylinReducer {
for (Text value : values) {
if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
-logger.info("Handling value with ordinal: " + vcounter);
+logger.info("Handling value with ordinal (This is not KV
number!): " + vcounter);
}
codec.decode(ByteBuffer.wrap(value.getBytes(), 0,
value.getLength()), input);
if (cuboidLevel > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index 04c9e90..244889f 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -80,7 +80,7 @@ public class InMemCuboidReducer extends
KylinReducerhttp://git-wip-us.apache.org/repos/asf/kylin/blob/020cf5f0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
new file mode 100644
index 000..713a95c
--- /dev/null
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed
[48/50] [abbrv] kylin git commit: KYLIN-2340 fix check style
KYLIN-2340 fix check style
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdcb9b81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdcb9b81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdcb9b81
Branch: refs/heads/master-hbase1.x
Commit: fdcb9b81d49d81a66cd290837d0dd89f0a308018
Parents: 9d3573a
Author: lidongsjtu
Authored: Fri Dec 30 16:02:34 2016 +0800
Committer: lidongsjtu
Committed: Fri Dec 30 16:02:34 2016 +0800
--
.../apache/kylin/query/relnode/OLAPToEnumerableConverter.java | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdcb9b81/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 405a24e..8f01a74 100644
---
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -42,9 +42,7 @@ import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.ModelChooser;
@@ -82,10 +80,10 @@ public class OLAPToEnumerableConverter extends
ConverterImpl implements Enumerab
// identify model
List contexts = listContextsHavingScan();
Set candidates = ModelChooser.selectModel(contexts);
-
+
// identify realization for each context
for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
-
+
// Context has no table scan is created by OLAPJoinRel which looks
like
// (sub-query) as A join (sub-query) as B
// No realization needed for such context.
[05/50] [abbrv] kylin git commit: KYLIN-2314 make CI cube snowflake and add test queries
KYLIN-2314 make CI cube snowflake and add test queries
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73fbbb90
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73fbbb90
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73fbbb90
Branch: refs/heads/master-hbase1.x
Commit: 73fbbb90e7cb0618bd27e146c89463cfcfb97e4b
Parents: c6c6719
Author: Yang Li
Authored: Sat Dec 24 15:34:48 2016 +0800
Committer: Yang Li
Committed: Sat Dec 24 16:47:17 2016 +0800
--
.../kylin/provision/BuildCubeWithStream.java| 10 +++--
.../apache/kylin/query/ITKylinQueryTest.java| 5 +
.../resources/query/sql_snowflake/query01.sql | 23
.../resources/query/sql_snowflake/query02.sql | 13 +++
.../resources/query/sql_snowflake/query03.sql | 14
.../resources/query/sql_snowflake/query04.sql | 19
.../resources/query/sql_snowflake/query05.sql | 20 +
.../resources/query/sql_snowflake/query06.sql | 21 ++
8 files changed, 123 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/73fbbb90/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 8abb84c..194ea35 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -311,6 +311,9 @@ public class BuildCubeWithStream {
}
public static void main(String[] args) throws Exception {
+long start = System.currentTimeMillis();
+int exitCode = 0;
+
BuildCubeWithStream buildCubeWithStream = null;
try {
beforeClass();
@@ -320,11 +323,14 @@ public class BuildCubeWithStream {
logger.info("Build is done");
buildCubeWithStream.cleanup();
logger.info("Going to exit");
-System.exit(0);
} catch (Throwable e) {
logger.error("error", e);
-System.exit(1);
+exitCode = 1;
}
+long millis = System.currentTimeMillis() - start;
+System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " +
BuildCubeWithStream.class.getName());
+
+System.exit(exitCode);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/73fbbb90/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index e86afa9..4329141 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -189,6 +189,11 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
+public void testSnowflakeQuery() throws Exception {
+execAndCompQuery(getQueryFolderPrefix() +
"src/test/resources/query/sql_snowflake", null, true);
+}
+
+@Test
public void testDateTimeQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() +
"src/test/resources/query/sql_datetime", null, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/73fbbb90/kylin-it/src/test/resources/query/sql_snowflake/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_snowflake/query01.sql
b/kylin-it/src/test/resources/query/sql_snowflake/query01.sql
new file mode 100644
index 000..e37d8f6
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_snowflake/query01.sql
@@ -0,0 +1,23 @@
+SELECT
+
+count(*) as cnt
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN EDW.TEST_CAL_DT as TEST_CAL_DT
+ON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND
TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN EDW.TEST_SITES as TEST_SITES
+ON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID
+INNER JOIN EDW.TEST_SELLER_TYPE_DIM as TEST_SELLER_TYPE_DIM
+ON T
[21/50] [abbrv] kylin git commit: KYLIN-2327 enable check-style for test code
KYLIN-2327 enable check-style for test code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e0edc84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e0edc84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e0edc84
Branch: refs/heads/master-hbase1.x
Commit: 3e0edc842ec4f6f7d9ef04dfa26afce101d2fd6e
Parents: 1563eb9
Author: Billy Liu
Authored: Wed Dec 28 01:16:35 2016 +0800
Committer: Billy Liu
Committed: Wed Dec 28 01:16:58 2016 +0800
--
.../apache/kylin/common/KylinConfigTest.java| 6 +-
.../org/apache/kylin/common/util/BasicTest.java | 2 +-
.../apache/kylin/common/util/JacksonBean.java | 7 +-
.../apache/kylin/common/util/JacksonTest.java | 5 +-
.../org/apache/kylin/cube/CubeDescTest.java | 6 +-
.../org/apache/kylin/cube/CubeManagerTest.java | 16 +-
.../inmemcubing/ConcurrentDiskStoreTest.java| 27 +-
.../cube/inmemcubing/MemDiskStoreTest.java | 27 +-
.../inmemcubing/MemoryBudgetControllerTest.java | 27 +-
.../validation/rule/DictionaryRuleTest.java | 1 -
.../gridtable/AggregationCacheMemSizeTest.java | 42 +-
.../kylin/gridtable/SimpleGridTableTest.java| 27 +-
.../gridtable/SimpleInvertedIndexTest.java | 27 +-
.../metadata/measure/MeasureCodecTest.java | 6 +-
.../kylin/dict/AppendTrieDictionaryTest.java| 3 +-
.../apache/kylin/dict/CachedTreeMapTest.java| 21 +-
.../kylin/dict/DictionaryManagerTest.java | 3 +-
.../kylin/dict/TrieDictionaryForestTest.java| 2 +-
.../kylin/dict/lookup/LookupTableTest.java | 1 -
.../impl/threadpool/DefaultSchedulerTest.java | 32 +-
.../measure/AggregatorMemEstimateTest.java | 17 +-
.../kylin/measure/hllc/HLLCounterOldTest.java | 3 +-
.../kylin/measure/hllc/HLLCounterTest.java | 13 +-
.../measure/topn/TopNCounterBasicTest.java | 16 +-
.../topn/TopNCounterCombinationTest.java| 16 +-
.../measure/topn/TopNCounterSerializerTest.java | 2 -
.../kylin/measure/topn/TopNCounterTest.java | 16 +-
.../kylin/metadata/MetadataManagerTest.java | 4 +-
.../DefaultPartitionConditionBuilderTest.java | 24 +-
.../apache/kylin/storage/StorageMockUtils.java | 33 +-
.../storage/gtrecord/DictGridTableTest.java | 2 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 2 +-
.../kylin/engine/mr/steps/MockupMapContext.java | 522 ++-
.../mr/steps/NumberDictionaryForestTest.java| 18 +
.../mr/steps/SelfDefineSortableKeyTest.java | 18 +
.../apache/kylin/jdbc/SQLResonseStubTest.java | 6 +-
.../inmemcubing/ITDoggedCubeBuilderTest.java| 30 +-
.../inmemcubing/ITInMemCubeBuilderTest.java | 30 +-
pom.xml | 4 +-
.../apache/kylin/query/udf/VersionUDFTest.java | 18 +
.../PasswordPlaceHolderConfigurerTest.java | 1 -
.../rest/controller/CubeControllerTest.java | 1 -
.../config/KafkaConsumerPropertiesTest.java | 18 +
.../hbase/common/HiveMiniClusterTest.java | 131 -
.../storage/hbase/steps/MockupMapContext.java | 522 ++-
45 files changed, 872 insertions(+), 883 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 209986f..a426fc6 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -18,7 +18,11 @@
package org.apache.kylin.common;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 9105245..bbaab09 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -82,7 +82,7 @@ public class BasicTest {
@Test
public void testxx() throws InterruptedException {
-System.out.println( 0x8fL);
+System.out.println(0x8fL);
byte
[31/50] [abbrv] kylin git commit: minor, fix NPE when filter Non-Cubing Job
minor, fix NPE when filter Non-Cubing Job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd745c88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd745c88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd745c88
Branch: refs/heads/master-hbase1.x
Commit: fd745c8804848a60ac96eff1bd9b30bbe38bdafc
Parents: 73a7c25
Author: Billy Liu
Authored: Wed Dec 28 18:47:58 2016 +0800
Committer: Billy Liu
Committed: Wed Dec 28 22:00:25 2016 +0800
--
.../main/java/org/apache/kylin/rest/service/JobService.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd745c88/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index ca8659c..4709a91 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -536,10 +536,13 @@ public class JobService extends BasicService implements
InitializingBean {
@Override
public boolean apply(AbstractExecutable executable) {
if (executable instanceof CubingJob) {
-if (cubeName == null) {
+if (StringUtils.isEmpty(cubeName)) {
return true;
}
-return
CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName);
+String executableCubeName =
CubingExecutableUtil.getCubeName(executable.getParams());
+if (executableCubeName == null)
+return true;
+return executableCubeName.contains(cubeName);
} else {
return false;
}
[29/50] [abbrv] kylin git commit: minor fix review bug
minor fix review bug Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76e321b2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76e321b2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76e321b2 Branch: refs/heads/master-hbase1.x Commit: 76e321b2d95955959fe3fac030195bbd321f9659 Parents: bc1a49e Author: zhongjian Authored: Wed Dec 28 15:22:02 2016 +0800 Committer: zhongjian Committed: Wed Dec 28 15:22:02 2016 +0800 -- webapp/app/partials/cubes/cube_clone.html | 4 ++-- webapp/app/partials/models/model_clone.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/76e321b2/webapp/app/partials/cubes/cube_clone.html -- diff --git a/webapp/app/partials/cubes/cube_clone.html b/webapp/app/partials/cubes/cube_clone.html index f5a332c..a2d3ad1 100644 --- a/webapp/app/partials/cubes/cube_clone.html +++ b/webapp/app/partials/cubes/cube_clone.html @@ -39,8 +39,8 @@ New Cube Name: -Cube name is required. -Cube name is invalid. +Cube name is required. +Cube name is invalid. http://git-wip-us.apache.org/repos/asf/kylin/blob/76e321b2/webapp/app/partials/models/model_clone.html -- diff --git a/webapp/app/partials/models/model_clone.html b/webapp/app/partials/models/model_clone.html index 484821a..2554633 100644 --- a/webapp/app/partials/models/model_clone.html +++ b/webapp/app/partials/models/model_clone.html @@ -50,8 +50,8 @@ New Model Name: -Model name is required. -Model name is invalid. +Model name is required. +Model name is invalid.
[43/50] [abbrv] kylin git commit: KYLIN-2328 minor review, open protected dumpKylinPropsAndMetadata for sub-class
KYLIN-2328 minor review, open protected dumpKylinPropsAndMetadata for sub-class
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cbef32f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cbef32f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cbef32f
Branch: refs/heads/master-hbase1.x
Commit: 5cbef32fcd51bffe31f92a985bb5d02405fb7b7c
Parents: 75c8d13
Author: Yang Li
Authored: Fri Dec 30 06:57:42 2016 +0800
Committer: Yang Li
Committed: Fri Dec 30 06:57:42 2016 +0800
--
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cbef32f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 4693ac3..e4e5e82 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -451,11 +451,11 @@ public abstract class AbstractHadoopJob extends
Configured implements Tool {
protected void attachTableMetadata(TableDesc table, Configuration conf)
throws IOException {
Set dumpList = new LinkedHashSet<>();
dumpList.add(table.getResourcePath());
-attachKylinPropsAndMetadata(dumpList,
KylinConfig.getInstanceFromEnv(), conf);
+dumpKylinPropsAndMetadata(dumpList, KylinConfig.getInstanceFromEnv(),
conf);
}
protected void attachCubeMetadata(CubeInstance cube, Configuration conf)
throws IOException {
-attachKylinPropsAndMetadata(collectCubeMetadata(cube),
cube.getConfig(), conf);
+dumpKylinPropsAndMetadata(collectCubeMetadata(cube), cube.getConfig(),
conf);
}
protected void attachCubeMetadataWithDict(CubeInstance cube, Configuration
conf) throws IOException {
@@ -464,14 +464,14 @@ public abstract class AbstractHadoopJob extends
Configured implements Tool {
for (CubeSegment segment : cube.getSegments()) {
dumpList.addAll(segment.getDictionaryPaths());
}
-attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
+dumpKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
}
protected void attachSegmentMetadataWithDict(CubeSegment segment,
Configuration conf) throws IOException {
Set dumpList = new LinkedHashSet<>();
dumpList.addAll(collectCubeMetadata(segment.getCubeInstance()));
dumpList.addAll(segment.getDictionaryPaths());
-attachKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
+dumpKylinPropsAndMetadata(dumpList, segment.getConfig(), conf);
}
private Set collectCubeMetadata(CubeInstance cube) {
@@ -490,7 +490,7 @@ public abstract class AbstractHadoopJob extends Configured
implements Tool {
return dumpList;
}
-private void attachKylinPropsAndMetadata(Set dumpList, KylinConfig
kylinConfig, Configuration conf) throws IOException {
+protected void dumpKylinPropsAndMetadata(Set dumpList, KylinConfig
kylinConfig, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
FileUtils.forceDelete(tmp); // we need a directory, so delete the file
first
[37/50] [abbrv] kylin git commit: upgrade pom to 2.0.0
upgrade pom to 2.0.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8cd416a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8cd416a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8cd416a3
Branch: refs/heads/master-hbase1.x
Commit: 8cd416a3b18c43a63a8437a8a35728d2ba25076e
Parents: 13c3407
Author: Li Yang
Authored: Thu Dec 29 17:03:23 2016 +0800
Committer: Li Yang
Committed: Thu Dec 29 17:03:23 2016 +0800
--
assembly/pom.xml | 2 +-
atopcalcite/pom.xml| 2 +-
core-common/pom.xml| 2 +-
.../src/main/java/org/apache/kylin/common/KylinVersion.java| 2 +-
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml| 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
pom.xml| 2 +-
query/pom.xml | 2 +-
server-base/pom.xml| 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml| 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
tomcat-ext/pom.xml | 2 +-
tool/pom.xml | 2 +-
22 files changed, 22 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 73f9e12..65e3a8c 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
kylin
org.apache.kylin
-1.6.1-SNAPSHOT
+2.0.0-SNAPSHOT
4.0.0
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index c1dba66..b1bc171 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -28,7 +28,7 @@
org.apache.kylin
kylin
-1.6.1-SNAPSHOT
+2.0.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 1bc5836..2265651 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,7 @@
org.apache.kylin
kylin
-1.6.1-SNAPSHOT
+2.0.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 622c6af..465ca5a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -69,7 +69,7 @@ public class KylinVersion {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
-private static final KylinVersion CURRENT_KYLIN_VERSION = new
KylinVersion("1.6.1");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new
KylinVersion("2.0.0");
private static final Set SIGNATURE_INCOMPATIBLE_REVISIONS =
new HashSet();
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index b013493..b7db363 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -28,7 +28,7 @@
org.apache.kylin
kylin
-1.6.1-SNAPSHOT
+2.0.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/8cd416a3/core-dictionary/pom.xml
---
[46/50] [abbrv] kylin git commit: KYLIN-2340 fix sub-query that creates two Filter nodes
KYLIN-2340 fix sub-query that creates two Filter nodes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e4637c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e4637c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e4637c9
Branch: refs/heads/master-hbase1.x
Commit: 6e4637c98814d8a3e8164dc366950bf8eb9ca716
Parents: 32d7244
Author: Li Yang
Authored: Fri Dec 30 15:17:44 2016 +0800
Committer: Li Yang
Committed: Fri Dec 30 15:17:44 2016 +0800
--
.../kylin/metadata/filter/TupleFilter.java | 23 +++
.../resources/query/sql_subquery/query14.sql| 22 ++
.../kylin/query/relnode/OLAPFilterRel.java | 5 ++--
.../relnode/OLAPToEnumerableConverter.java | 24 +---
4 files changed, 49 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e4637c9/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 31de50b..f9d83f5 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.slf4j.Logger;
@@ -241,4 +242,26 @@ public abstract class TupleFilter {
}
}
+public static TupleFilter and(TupleFilter f1, TupleFilter f2) {
+if (f1 == null)
+return f2;
+if (f2 == null)
+return f1;
+
+if (f1.getOperator() == FilterOperatorEnum.AND) {
+f1.addChild(f2);
+return f1;
+}
+
+if (f2.getOperator() == FilterOperatorEnum.AND) {
+f2.addChild(f1);
+return f2;
+}
+
+LogicalTupleFilter and = new
LogicalTupleFilter(FilterOperatorEnum.AND);
+and.addChild(f1);
+and.addChild(f2);
+return and;
+}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e4637c9/kylin-it/src/test/resources/query/sql_subquery/query14.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query14.sql
b/kylin-it/src/test/resources/query/sql_subquery/query14.sql
new file mode 100644
index 000..b3b3b7e
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query14.sql
@@ -0,0 +1,22 @@
+select
+lstg_format_name,
+sum(price) as gvm
+from
+(
+select
+cal_dt,
+lstg_format_name,
+price
+from test_kylin_fact
+inner JOIN test_category_groupings
+ON test_kylin_fact.leaf_categ_id =
test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id =
test_category_groupings.site_id
+inner JOIN edw.test_sites as test_sites
+ON test_kylin_fact.lstg_site_id = test_sites.site_id
+where
+lstg_site_id = 0
+and cal_dt > '2013-05-13'
+) f
+where
+lstg_format_name ='Auction'
+group by
+lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e4637c9/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 151131e..d3d78a0 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -322,9 +322,10 @@ public class OLAPFilterRel extends Filter implements
OLAPRel {
}
TupleFilterVisitor visitor = new
TupleFilterVisitor(this.columnRowType, context);
-context.filter = this.condition.accept(visitor);
+TupleFilter filter = this.condition.accept(visitor);
-context.filterColumns = visitor.columnsInFilter;
+context.filter = TupleFilter.and(context.filter, filter);
+context.filterColumns.addAll(visitor.columnsInFilter);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e4637c9/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
---
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
http://git-wip-us.apache.org/repos/asf/kylin/blob/982e8cf2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed
with coprocessor.");
} catch (Exception ex) {
-logger.error("Error deploying coprocessor on " +
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " +
tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
-public static boolean resetCoprocessor(String tableName, HBaseAdmin
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin,
Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc =
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws
IOException {
+private static List resetCoprocessorOnHTables(final Admin
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws
IOException {
List processedTables = Collections.synchronizedList(new
ArrayList());
ExecutorService coprocessorPool =
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List processedTables;
-public ResetCoprocessorWorker(CountDownLatch countDownLatch,
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch
[23/50] [abbrv] kylin git commit: fix NPE in OLAPEnumerator.setConnectionProperties()
fix NPE in OLAPEnumerator.setConnectionProperties()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cccf3a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cccf3a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cccf3a9c
Branch: refs/heads/master-hbase1.x
Commit: cccf3a9c1a8dd0c7ac326c958aafab050a3886cb
Parents: 84ec908
Author: Li Yang
Authored: Wed Dec 28 09:45:18 2016 +0800
Committer: Li Yang
Committed: Wed Dec 28 09:45:18 2016 +0800
--
.../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/cccf3a9c/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index e4fc937..f012638 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,6 +25,7 @@ import java.util.Properties;
import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -161,8 +162,10 @@ public class OLAPEnumerator implements
Enumerator {
Properties connProps = conn.getProperties();
String propThreshold =
connProps.getProperty(OLAPQuery.PROP_SCAN_THRESHOLD);
-int threshold = Integer.valueOf(propThreshold);
-olapContext.storageContext.setThreshold(threshold);
+if (!StringUtils.isBlank(propThreshold)) {
+int threshold = Integer.valueOf(propThreshold);
+olapContext.storageContext.setThreshold(threshold);
+}
}
}
[38/50] [abbrv] kylin git commit: enhance LookupTable
enhance LookupTable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64c3c618
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64c3c618
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64c3c618
Branch: refs/heads/master-hbase1.x
Commit: 64c3c61898d10f05c4350609517b2096c4171167
Parents: 8cd416a
Author: Li Yang
Authored: Thu Dec 29 13:44:40 2016 +0800
Committer: Li Yang
Committed: Thu Dec 29 18:26:59 2016 +0800
--
.../src/main/java/org/apache/kylin/dict/lookup/LookupTable.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/64c3c618/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index cd700e9..21221f4 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -46,7 +47,7 @@ abstract public class LookupTable {
protected TableDesc tableDesc;
protected String[] keyColumns;
protected ReadableTable table;
-protected ConcurrentHashMap, T[]> data;
+protected Map, T[]> data;
public LookupTable(TableDesc tableDesc, String[] keyColumns, ReadableTable
table) throws IOException {
this.tableDesc = tableDesc;
[35/50] [abbrv] kylin git commit: KYLIN-2333 Kylin doesn't need 0-D cuboid, can remove that step
KYLIN-2333 Kylin doesn't need 0-D cuboid, can remove that step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/687d5fd1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/687d5fd1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/687d5fd1
Branch: refs/heads/master-hbase1.x
Commit: 687d5fd150b6019e71db0f600307bbd4f7c749f9
Parents: 1a190ec
Author: shaofengshi
Authored: Thu Dec 29 13:23:14 2016 +0800
Committer: shaofengshi
Committed: Thu Dec 29 13:23:14 2016 +0800
--
.../java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/687d5fd1/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 6eba3c2..966bb1b 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -84,7 +84,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport
{
// base cuboid step
result.addTask(createBaseCuboidStep(cuboidOutputTempPath, jobId));
// n dim cuboid steps
-for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
+for (int i = 1; i < groupRowkeyColumnsCount; i++) {
int dimNum = totalRowkeyColumnsCount - i;
result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath,
dimNum, totalRowkeyColumnsCount, jobId));
}
[33/50] [abbrv] kylin git commit: minor, get config by prefix in KylinConfigCLI
minor, get config by prefix in KylinConfigCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdecf093
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdecf093
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdecf093
Branch: refs/heads/master-hbase1.x
Commit: fdecf0937802a2d17418595e6c0543c9668ac8a4
Parents: 08eda7b
Author: lidongsjtu
Authored: Wed Dec 28 22:17:35 2016 +0800
Committer: lidongsjtu
Committed: Wed Dec 28 22:45:57 2016 +0800
--
.../test_case_data/localmeta/kylin.properties | 3 +
.../org/apache/kylin/tool/KylinConfigCLI.java | 30 ++--
.../apache/kylin/tool/KylinConfigCLITest.java | 73
3 files changed, 102 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/examples/test_case_data/localmeta/kylin.properties
--
diff --git a/examples/test_case_data/localmeta/kylin.properties
b/examples/test_case_data/localmeta/kylin.properties
index 1dfac32..f4c6772 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -131,3 +131,6 @@ kylin.test.bcc.new.key=some-value
kylin.engine.mr.config-override.test1=test1
kylin.engine.mr.config-override.test2=test2
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
+
+kylin.engine.provider.0=org.apache.kylin.engine.mr.MRBatchCubingEngine
+kylin.cube.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
index e1a5b99..b740beb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -18,11 +18,14 @@
package org.apache.kylin.tool;
+import java.util.Map;
import java.util.Properties;
import org.apache.kylin.common.BackwardCompatibilityConfig;
import org.apache.kylin.common.KylinConfig;
+import com.google.common.collect.Maps;
+
public class KylinConfigCLI {
public static void main(String[] args) {
if (args.length != 1) {
@@ -33,10 +36,29 @@ public class KylinConfigCLI {
Properties config = KylinConfig.getKylinProperties();
BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
-String value = config.getProperty(bcc.check(args[0]));
-if (value == null) {
-value = "";
+String key = bcc.check(args[0].trim());
+if (!key.endsWith(".")) {
+String value = config.getProperty(key);
+if (value == null) {
+value = "";
+}
+System.out.println(value);
+} else {
+Map props = getPropertiesByPrefix(config, key);
+for (Map.Entry prop : props.entrySet()) {
+System.out.println(prop.getKey() + "=" + prop.getValue());
+}
+}
+}
+
+static private Map getPropertiesByPrefix(Properties props,
String prefix) {
+Map result = Maps.newLinkedHashMap();
+for (Map.Entry entry : props.entrySet()) {
+String entryKey = (String) entry.getKey();
+if (entryKey.startsWith(prefix)) {
+result.put(entryKey.substring(prefix.length()), (String)
entry.getValue());
+}
}
-System.out.println(value);
+return result;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdecf093/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
--
diff --git a/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
new file mode 100644
index 000..7d1e248
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/KylinConfigCLITest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS
[09/50] [abbrv] kylin git commit: KYLIN-2322 make TrieDictionaryForest configurable, off by default
KYLIN-2322 make TrieDictionaryForest configurable, off by default
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a78b6509
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a78b6509
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a78b6509
Branch: refs/heads/master-hbase1.x
Commit: a78b6509f8727a53e26c57b2e5a14adc0ff88b73
Parents: 4f466a0
Author: Yang Li
Authored: Mon Dec 26 21:12:20 2016 +0800
Committer: Yang Li
Committed: Mon Dec 26 21:14:01 2016 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 ++
.../apache/kylin/dict/DictionaryGenerator.java | 64 ++--
2 files changed, 63 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index da93388..8080577 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,6 +216,10 @@ abstract public class KylinConfigBase implements
Serializable {
// DICTIONARY & SNAPSHOT
//
+public boolean isUseForestTrieDictionary() {
+return
Boolean.parseBoolean(getOptional("kylin.dictionary.use-forest-trie", "false"));
+}
+
public int getTrieDictionaryForestMaxTrieSizeMB() {
return
Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index cd13d59..6f4f2c4 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.datatype.DataType;
@@ -49,10 +50,12 @@ public class DictionaryGenerator {
builder = new DateDictBuilder();
else
builder = new TimeDictBuilder();
-} else if (dataType.isNumberFamily()) {
-builder = new NumberDictBuilder();
} else {
-builder = new StringDictBuilder();
+boolean useForest =
KylinConfig.getInstanceFromEnv().isUseForestTrieDictionary();
+if (dataType.isNumberFamily())
+builder = useForest ? new NumberTrieDictForestBuilder() : new
NumberTrieDictBuilder();
+else
+builder = useForest ? new StringTrieDictForestBuilder() : new
StringTrieDictBuilder();
}
return builder;
}
@@ -168,7 +171,32 @@ public class DictionaryGenerator {
}
}
-private static class StringDictBuilder implements IDictionaryBuilder {
+private static class StringTrieDictBuilder implements IDictionaryBuilder {
+int baseId;
+TrieDictionaryBuilder builder;
+
+@Override
+public void init(DictionaryInfo info, int baseId) throws IOException {
+this.baseId = baseId;
+this.builder = new TrieDictionaryBuilder(new
StringBytesConverter());
+}
+
+@Override
+public boolean addValue(String value) {
+if (value == null)
+return false;
+
+builder.addValue(value);
+return true;
+}
+
+@Override
+public Dictionary build() throws IOException {
+return builder.build(baseId);
+}
+}
+
+private static class StringTrieDictForestBuilder implements
IDictionaryBuilder {
TrieDictionaryForestBuilder builder;
@Override
@@ -191,7 +219,33 @@ public class DictionaryGenerator {
}
}
-private static class NumberDictBuilder implements IDictionaryBuilder {
+@SuppressWarnings("deprecation")
+private static class NumberTrieDictBuilder implements IDictionaryBuilder {
+int baseId;
+NumberDictionaryBuilder builder;
+
+@Ove
[32/50] [abbrv] kylin git commit: minor, fix could not parse issue for scheduler
minor, fix could not parse issue for scheduler
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/08eda7b7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/08eda7b7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/08eda7b7
Branch: refs/heads/master-hbase1.x
Commit: 08eda7b714a30c665bdd777c296d15c7650c9ea9
Parents: fd745c8
Author: Billy Liu
Authored: Wed Dec 28 19:00:00 2016 +0800
Committer: Billy Liu
Committed: Wed Dec 28 22:01:39 2016 +0800
--
.../kylin/job/impl/threadpool/DefaultScheduler.java | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/08eda7b7/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index be8e7fe..ddb3a21 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -105,15 +105,18 @@ public class DefaultScheduler implements
Scheduler, Connecti
continue;
}
nReady++;
-AbstractExecutable executable =
executableManager.getJob(id);
-String jobDesc = executable.toString();
-logger.info(jobDesc + " prepare to schedule");
+AbstractExecutable executable = null;
+String jobDesc = null;
try {
+executable = executableManager.getJob(id);
+jobDesc = executable.toString();
+logger.info(jobDesc + " prepare to schedule");
context.addRunningJob(executable);
jobPool.execute(new JobRunner(executable));
logger.info(jobDesc + " scheduled");
} catch (Exception ex) {
-context.removeRunningJob(executable);
+if (executable != null)
+context.removeRunningJob(executable);
logger.warn(jobDesc + " fail to schedule", ex);
}
}
[11/50] [abbrv] kylin git commit: minor, more stable DeployUtil
minor, more stable DeployUtil
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c2d6bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c2d6bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c2d6bc
Branch: refs/heads/master-hbase1.x
Commit: 74c2d6bcf0ea96939c63c3432961d666c77e609c
Parents: f9a8402
Author: Yang Li
Authored: Tue Dec 27 07:02:32 2016 +0800
Committer: Yang Li
Committed: Tue Dec 27 07:02:32 2016 +0800
--
assembly/src/test/java/org/apache/kylin/job/DeployUtil.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/74c2d6bc/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 089c3ed..a42b125 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -255,8 +255,9 @@ public class DeployUtil {
private static String[] generateCreateTableHql(TableDesc tableDesc) {
String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
+String dropsql2 = "DROP VIEW IF EXISTS " + tableDesc.getIdentity();
+
StringBuilder ddl = new StringBuilder();
-
ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
ddl.append("(" + "\n");
@@ -272,7 +273,7 @@ public class DeployUtil {
ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
ddl.append("STORED AS TEXTFILE");
-return new String[] { dropsql, ddl.toString() };
+return new String[] { dropsql, dropsql2, ddl.toString() };
}
private static String[] generateCreateViewHql(String viewName, String
tableName) {
[02/50] [abbrv] kylin git commit: KYLIN-2320 fix getSizeOfId() for empty TrieDictionaryForest
KYLIN-2320 fix getSizeOfId() for empty TrieDictionaryForest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30d3763f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30d3763f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30d3763f
Branch: refs/heads/master-hbase1.x
Commit: 30d3763f8256a9e88ae9620499021e99771b3351
Parents: 73a4029
Author: Yang Li
Authored: Sat Dec 24 07:34:12 2016 +0800
Committer: Yang Li
Committed: Sat Dec 24 14:46:38 2016 +0800
--
.../apache/kylin/dict/TrieDictionaryForest.java | 4 +-
.../kylin/dict/TrieDictionaryForestTest.java| 41
2 files changed, 35 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d3763f/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 1015b11..e746348 100755
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -87,7 +87,7 @@ public class TrieDictionaryForest extends Dictionary {
@Override
public int getSizeOfId() {
if (trees.isEmpty())
-return -1;
+return 1;
int maxOffset = accuOffset.get(accuOffset.size() - 1);
TrieDictionary lastTree = trees.get(trees.size() - 1);
int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset +
lastTree.getMaxId() + 1);
@@ -96,7 +96,7 @@ public class TrieDictionaryForest extends Dictionary {
@Override
public int getSizeOfValue() {
-int maxValue = -1;
+int maxValue = 0;
for (TrieDictionary tree : trees)
maxValue = Math.max(maxValue, tree.getSizeOfValue());
return maxValue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d3763f/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
--
diff --git
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 32a6ab5..238bb31 100755
---
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -52,6 +52,14 @@ import org.junit.Test;
public class TrieDictionaryForestTest {
@Test
+public void testEmptyDict() {
+ArrayList strs = new ArrayList();
+TrieDictionaryForestBuilder builder = newDictBuilder(strs, 0);
+TrieDictionaryForest dict = builder.build();
+assertSameBehaviorAsTrie(dict, strs, 0);
+}
+
+@Test
public void testBasicFound() {
ArrayList strs = new ArrayList();
strs.add("part");
@@ -67,11 +75,11 @@ public class TrieDictionaryForestTest {
dict.dump(System.out);
int expectId = baseId;
for (String s : strs) {
-System.out.println("value:" + s + " expect id:" + expectId);
assertEquals(expectId, dict.getIdFromValue(s));
expectId++;
}
-System.out.println("test ok");
+
+assertSameBehaviorAsTrie(dict, strs, baseId);
}
@Test //one string one tree
@@ -95,11 +103,11 @@ public class TrieDictionaryForestTest {
assertEquals(strs.size(), dict.getTrees().size());
int expectId = baseId;
for (String s : strs) {
-System.out.println("value:" + s + " expect id:" + expectId);
assertEquals(expectId, dict.getIdFromValue(s));
expectId++;
}
-System.out.println("test ok");
+
+assertSameBehaviorAsTrie(dict, strs, baseId);
}
@Test
@@ -115,11 +123,13 @@ public class TrieDictionaryForestTest {
dict.dump(System.out);
//null value query
int id = dict.getIdFromValue(null, 0);
-System.out.println(id);
+assertEquals(255, id);
id = dict.getIdFromValue(null, 1);
-System.out.println(id);
+assertEquals(255, id);
id = dict.getIdFromValue(null, -1);
-System.out.println(id);
+assertEquals(255, id);
+
+assertSameBehaviorAsTrie(dict, strs, 0);
}
@Test
@@ -940,7 +950,7 @@ public class TrieDictionaryForestTest {
b.addValue(strs.next());
return b;
}
-
+
private static class RandomStrings implements Iterable {
final private int size;
@@ -1022,4 +10
[13/50] [abbrv] kylin git commit: strenthen storage visit deadline calculation
strenthen storage visit deadline calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b6c8c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b6c8c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b6c8c1
Branch: refs/heads/master-hbase1.x
Commit: 53b6c8c18033f92feadd2295456fa4f31011b5bb
Parents: 2436140
Author: Hongbin Ma
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:15:44 2016 +0800
--
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/53b6c8c1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() +
scanReq.getTimeout();
logger.info("deadline is " + deadline);
final long storagePushDownLimit =
scanReq.getStoragePushDownLimit();
[27/50] [abbrv] kylin git commit: minor, refine rootfact select show width database
minor,refine rootfact select show width database Signed-off-by: zhongjian Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94acbc71 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94acbc71 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94acbc71 Branch: refs/heads/master-hbase1.x Commit: 94acbc7130bb9084af9b5ac6def2527c5d238b15 Parents: 8747e24 Author: luguosheng <550175...@qq.com> Authored: Wed Dec 28 11:39:05 2016 +0800 Committer: zhongjian Committed: Wed Dec 28 15:09:17 2016 +0800 -- webapp/app/partials/modelDesigner/data_model.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/94acbc71/webapp/app/partials/modelDesigner/data_model.html -- diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html index 6e66e91..4d72266 100644 --- a/webapp/app/partials/modelDesigner/data_model.html +++ b/webapp/app/partials/modelDesigner/data_model.html @@ -27,7 +27,7 @@
[28/50] [abbrv] kylin git commit: minor,update model designer
minor,update model designer
Signed-off-by: zhongjian
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc1a49ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc1a49ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc1a49ec
Branch: refs/heads/master-hbase1.x
Commit: bc1a49ec57987cf52d2596ac4d7a439b7596eb0d
Parents: 94acbc7
Author: chenzhx <346839...@qq.com>
Authored: Wed Dec 28 11:11:08 2016 +0800
Committer: zhongjian
Committed: Wed Dec 28 15:15:36 2016 +0800
--
webapp/app/js/controllers/modelDataModel.js | 5
webapp/app/js/controllers/modelSchema.js| 29 ++--
webapp/app/js/services/tree.js | 20 --
webapp/app/partials/cubes/cube_clone.html | 23
.../partials/modelDesigner/model_measures.html | 1 +
webapp/app/partials/models/model_clone.html | 10 ---
6 files changed, 20 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/controllers/modelDataModel.js
--
diff --git a/webapp/app/js/controllers/modelDataModel.js
b/webapp/app/js/controllers/modelDataModel.js
index 95c42c4..07009a2 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -203,12 +203,17 @@ KylinApp.controller('ModelDataModelCtrl', function
($location,$scope, $modal,cub
modelsManager.selectedModel.dimensions.splice(i,
1);
}
}
+delete $scope.aliasTableMap[lookup.alias];
lookupList.splice(lookupList.indexOf(lookup), 1);
+
$scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
}
});
}else{
+delete $scope.aliasTableMap[lookup.alias];
lookupList.splice(lookupList.indexOf(lookup), 1);
+$scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
}
+console.log($scope.aliasName);
};
$scope.changeKey = function(index){
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/controllers/modelSchema.js
--
diff --git a/webapp/app/js/controllers/modelSchema.js
b/webapp/app/js/controllers/modelSchema.js
index e309873..f917f1e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -249,35 +249,10 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope,
QueryService, UserServi
* 1.metric can't be null
*/
$scope.check_model_measure = function () {
-
-var errors = [];
-if (!modelsManager.selectedModel.metrics ||
!modelsManager.selectedModel.metrics.length) {
- errors.push("Please define your metrics.");
-}
-var errorInfo = "";
-angular.forEach(errors, function (item) {
- errorInfo += "\n" + item;
-});
-if (errors.length) {
- SweetAlert.swal('', errorInfo, 'warning');
- return false;
-} else {
- return true;
-}
-
+return true;
};
$scope.check_model_setting = function () {
-var errors = [];
-var errorInfo = "";
-angular.forEach(errors, function (item) {
- errorInfo += "\n" + item;
-});
-if (errors.length) {
- SweetAlert.swal('', errorInfo, 'warning');
- return false;
-} else {
- return true;
-}
+return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/services/tree.js
--
diff --git a/webapp/app/js/services/tree.js b/webapp/app/js/services/tree.js
index 20de19b..921aefe 100755
--- a/webapp/app/js/services/tree.js
+++ b/webapp/app/js/services/tree.js
@@ -72,27 +72,7 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
}
});
-/* angular.forEach(model.dimensions, function (dimension, index) {
-// for dimension on lookup table
-if(model.fact_table!==dimension.table){
-var lookup = _.find(graphData.children,function(item){
- return item.name === dimension.table;
-});
- angular.forEach(lookup.join.primary_key, function(pk, index){
- for (var i = 0; i < lookup._children.length; i++) {
- if(lookup._children[i].name == pk)
- break;
- }
- if(i == lookup._children.length) {
-lookup._children.push({
- "type": "column",
- "name": pk
- });
- }
-
[44/50] [abbrv] kylin git commit: KYLIN-2323 code review
KYLIN-2323 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9ee6d33
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9ee6d33
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9ee6d33
Branch: refs/heads/master-hbase1.x
Commit: a9ee6d33eca53f67d9d4dd1da52b6fbfe98cb510
Parents: 5cbef32
Author: Yang Li
Authored: Fri Dec 30 07:07:53 2016 +0800
Committer: Yang Li
Committed: Fri Dec 30 07:07:53 2016 +0800
--
.../java/org/apache/kylin/common/util/StringUtil.java| 11 +++
.../apache/kylin/rest/controller/TableController.java| 3 ++-
2 files changed, 13 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9ee6d33/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index bbc9448..12e7279 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -153,4 +153,15 @@ public class StringUtil {
}
}
+public static String[] splitAndTrim(String str, String splitBy) {
+String[] split = str.split(splitBy);
+ArrayList r = new ArrayList<>(split.length);
+for (String s : split) {
+s = s.trim();
+if (!s.isEmpty())
+r.add(s);
+}
+return (String[]) r.toArray(new String[r.size()]);
+}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9ee6d33/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index eed5413..1ab6ba6 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
@@ -94,7 +95,7 @@ public class TableController extends BasicController {
public Map loadHiveTables(@PathVariable String tables,
@PathVariable String project, @RequestBody HiveTableRequest request) throws
IOException {
String submitter =
SecurityContextHolder.getContext().getAuthentication().getName();
Map result = new HashMap();
-String[] tableNames = tables.split(",");
+String[] tableNames = StringUtil.splitAndTrim(tables, ",");
try {
String[] loaded = tableService.loadHiveTablesToProject(tableNames,
project);
result.put("result.loaded", loaded);
[15/50] [abbrv] kylin git commit: KYLIN-2318 query cache is not working
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d
Branch: refs/heads/master-hbase1.x
Commit: 182c565df6b2134ef1302c88988ed27b60fefa42
Parents: fa4a5ee
Author: Hongbin Ma
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:15:44 2016 +0800
--
.../org/apache/kylin/common/QueryContext.java | 61
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +---
.../apache/kylin/rest/service/QueryService.java | 23
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to
SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
public static Pair getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v == null) {
@@ -104,8 +105,6 @@ public class BackdoorToggles {
_backdoorToggles.remove();
}
-
[17/50] [abbrv] kylin git commit: close OLAPEnumerator at exception
close OLAPEnumerator at exception
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/596ab51b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/596ab51b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/596ab51b
Branch: refs/heads/master-hbase1.x
Commit: 596ab51b4b98ae0923a134ede56c0ceb2f23bb66
Parents: 53b6c8c
Author: Hongbin Ma
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:17:30 2016 +0800
--
.../kylin/query/enumerator/OLAPEnumerator.java | 33 +---
1 file changed, 22 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/596ab51b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,31 @@ public class OLAPEnumerator implements Enumerator
{
@Override
public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} catch (Exception e) {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
+}
+throw e;
}
-convertCurrentRow(tuple);
-return true;
}
private Object[] convertCurrentRow(ITuple tuple) {
[19/50] [abbrv] kylin git commit: KYLIN-2318 bug fix
KYLIN-2318 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1563eb94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1563eb94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1563eb94
Branch: refs/heads/master-hbase1.x
Commit: 1563eb945d0f3c68874934beca18579c96acc100
Parents: bfe8b1a
Author: Hongbin Ma
Authored: Tue Dec 27 12:47:08 2016 +0800
Committer: Hongbin Ma
Committed: Tue Dec 27 22:17:33 2016 +0800
--
.../org/apache/kylin/common/debug/BackdoorToggles.java| 10 ++
.../java/org/apache/kylin/rest/service/QueryService.java | 1 +
2 files changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1563eb94/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 8109cf2..95d5d62 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -23,6 +23,8 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
+import com.google.common.collect.Maps;
+
/**
* BackdoorToggles and QueryContext are similar because they're both hosting
per-query thread local variables.
* The difference is that BackdoorToggles are specified by user input and work
for debug purpose. QueryContext
@@ -38,6 +40,14 @@ public class BackdoorToggles {
_backdoorToggles.set(toggles);
}
+public static void addToggles(Map toggles) {
+Map map = _backdoorToggles.get();
+if (map == null) {
+setToggles(Maps. newHashMap());
+}
+_backdoorToggles.get().putAll(toggles);
+}
+
public static String getCoprocessorBehavior() {
return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1563eb94/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bb0342f..de28b17 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,6 +327,7 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
[04/50] [abbrv] kylin git commit: KYLIN-2314 Use col identity instead of col name in dictionary refs
KYLIN-2314 Use col identity instead of col name in dictionary refs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6c67197
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6c67197
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6c67197
Branch: refs/heads/master-hbase1.x
Commit: c6c6719730e05c66c14407cfd7926a0a54a6009a
Parents: 30d3763
Author: Li Yang
Authored: Fri Dec 23 18:06:58 2016 +0800
Committer: Yang Li
Committed: Sat Dec 24 14:46:39 2016 +0800
--
.../java/org/apache/kylin/job/DeployUtil.java | 26 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 30 +-
.../org/apache/kylin/cube/CubeDescTest.java | 40 +-
.../kylin/metadata/MetadataManagerTest.java | 67 ++-
.../kylin/metadata/model/JoinsTreeTest.java | 11 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 -
.../engine/mr/steps/CreateDictionaryJob.java| 5 +-
.../engine/mr/steps/CubingExecutableUtil.java | 8 -
.../engine/mr/steps/FactDistinctColumnsJob.java | 3 +
.../mr/steps/FactDistinctColumnsReducer.java| 39 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 81 ++--
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 1 -
.../localmeta/cube_desc/ci_inner_join_cube.json | 248 ---
.../localmeta/cube_desc/ci_left_join_cube.json | 248 ---
.../test_kylin_snowflake_sales_cube.json| 268
.../localmeta/data/DEFAULT.TEST_COUNTRY.csv | 244 +++
.../model_desc/ci_inner_join_model.json | 100 -
.../model_desc/ci_left_join_model.json | 100 -
.../test_kylin_snowflake_sales_model.json | 87
.../localmeta/table/DEFAULT.TEST_ACCOUNT.json | 34 ++
.../localmeta/table/DEFAULT.TEST_COUNTRY.json | 24 ++
.../table/DEFAULT.TEST_KYLIN_COUNTRY.json | 24 --
.../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 --
.../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 ---
.../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 ---
.../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 --
.../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 ---
.../java/org/apache/kylin/query/H2Database.java | 11 +-
30 files changed, 1080 insertions(+), 1295 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 564a4e2..089c3ed 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -125,11 +125,15 @@ public class DeployUtil {
static final String TABLE_CATEGORY_GROUPINGS =
"default.test_category_groupings";
static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
static final String TABLE_ORDER = "default.test_order";
+static final String TABLE_ACCOUNT = "default.test_account";
+static final String TABLE_COUNTRY = "default.test_country";
static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
static final String TABLE_SELLER_TYPE_DIM_TABLE =
"edw.test_seller_type_dim_table";
static final String TABLE_SITES = "edw.test_sites";
-static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT,
TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT,
TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
+static final String[] TABLE_NAMES = new String[] { //
+TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS,
TABLE_KYLIN_FACT, //
+TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES, TABLE_ACCOUNT,
TABLE_COUNTRY };
public static void prepareTestDataForNormalCubes(String modelName) throws
Exception {
@@ -228,21 +232,15 @@ public class DeployUtil {
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
// create hive tables
hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase(;
-
hiveClient.executeHQL(generateCrea
