KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d9a59be Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d9a59be Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d9a59be Branch: refs/heads/1.5.x-CDH5.7 Commit: 7d9a59be2cba4fb0cb08f8a2ee28b7217ea54f52 Parents: 5d679f1 Author: shaofengshi <shaofeng...@apache.org> Authored: Wed Mar 23 17:07:05 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Jul 26 09:49:00 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../kylin/provision/BuildCubeWithEngine.java | 2 ++ .../org/apache/kylin/rest/service/CubeService.java | 2 ++ .../org/apache/kylin/rest/service/UserService.java | 1 - .../kylin/storage/hbase/HBaseConnection.java | 1 - .../kylin/storage/hbase/HBaseResourceStore.java | 12 +++++++----- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 17 +++++++++++++++++ .../storage/hbase/steps/HBaseCuboidWriter.java | 16 ++++++++-------- .../storage/hbase/util/StorageCleanupJob.java | 1 - 9 files changed, 40 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b146031..87b7c22 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -781,4 +781,8 @@ abstract public class KylinConfigBase implements Serializable { public void setAppendDictCacheSize(int cacheSize) { setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize)); } + + public String getPatchedFuzzyRowFilterVersion() { + return this.getOptional("kylin.hbase.filter.fuzzy.row.filter.version", "1.1.3"); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index 2a5979f..30e95d3 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -58,6 +58,8 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.impl.threadpool.DefaultScheduler; import org.apache.kylin.job.manager.ExecutableManager; +import org.apache.kylin.metadata.model.IEngineAware; +import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.storage.hbase.HBaseConnection; import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator; import org.apache.kylin.storage.hbase.util.StorageCleanupJob; http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index 0503236..4e5d793 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -32,6 +32,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.HTable; import java.util.*; +import java.util.*; +import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeInstance; http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java index e039534..0e0e213 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java @@ -152,7 +152,6 @@ public class UserService implements UserDetailsManager { Put put = new Put(pair.getKey()); put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond()); - htable.put(put); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index 7dbc7f0..e7ee2f5 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -283,4 +283,3 @@ public class HBaseConnection { } } - http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index aa7a4d4..9881569 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -33,8 +33,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; @@ -324,9 +322,13 @@ public class HBaseResourceStore extends ResourceStore { get.addColumn(B_FAMILY, B_COLUMN_TS); } - Result result = table.get(get); - boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists())); - return exists ? result : null; + try { + Result result = table.get(get); + boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists())); + return exists ? result : null; + } finally { + IOUtils.closeQuietly(table); + } } private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index d84074f..9f787ad 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -435,4 +435,21 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { return sb.toString(); } + + private Map<byte[], CubeVisitProtos.CubeVisitResponse> getResults(final CubeVisitProtos.CubeVisitRequest request, Table table, byte[] startKey, byte[] endKey) throws Throwable { + Map<byte[], CubeVisitProtos.CubeVisitResponse> results = table.coprocessorService(CubeVisitProtos.CubeVisitService.class, startKey, endKey, new Batch.Call<CubeVisitProtos.CubeVisitService, CubeVisitProtos.CubeVisitResponse>() { + public CubeVisitProtos.CubeVisitResponse call(CubeVisitProtos.CubeVisitService rowsService) throws IOException { + ServerRpcController controller = new ServerRpcController(); + BlockingRpcCallback<CubeVisitProtos.CubeVisitResponse> rpcCallback = new BlockingRpcCallback<>(); + rowsService.visitCube(controller, request, rpcCallback); + CubeVisitProtos.CubeVisitResponse response = rpcCallback.get(); + if (controller.failedOnException()) { + throw controller.getFailedOn(); + } + return response; + } + }); + + return results; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java index c990379..a4a43c9 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java @@ -128,15 +128,15 @@ public class HBaseCuboidWriter implements ICuboidWriter { @Override public final void flush() throws IOException { - if (!puts.isEmpty()) { - long t = System.currentTimeMillis(); - if (hTable != null) { - hTable.put(puts); + + if (!puts.isEmpty()) { + long t = System.currentTimeMillis(); + if (hTable != null) { + hTable.put(puts); + } + logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms"); + puts.clear(); } - logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms"); - puts.clear(); - } - puts.clear(); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java index 874121d..06eafa9 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.CliCommandExecutor;
