Repository: kylin Updated Branches: refs/heads/master-hbase0.98 [created] 89ef67758
http://git-wip-us.apache.org/repos/asf/kylin/blob/89ef6775/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java index 42a54c8..e36f662 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java @@ -24,18 +24,16 @@ import java.util.Arrays; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,15 +49,14 @@ public class UpdateHTableHostCLI { private List<String> errorMsgs = Lists.newArrayList(); private List<String> htables; - private Admin hbaseAdmin; + private HBaseAdmin hbaseAdmin; private KylinConfig kylinConfig; private String oldHostValue; public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException { this.htables = htables; this.oldHostValue = oldHostValue; - Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create()); - hbaseAdmin = conn.getAdmin(); + this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration()); this.kylinConfig = KylinConfig.getInstanceFromEnv(); } @@ -169,9 +166,9 @@ public class UpdateHTableHostCLI { HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) { desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); - hbaseAdmin.disableTable(TableName.valueOf(tableName)); - hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc); - hbaseAdmin.enableTable(TableName.valueOf(tableName)); + hbaseAdmin.disableTable(tableName); + hbaseAdmin.modifyTable(tableName, desc); + hbaseAdmin.enableTable(tableName); updatedResources.add(tableName); } http://git-wip-us.apache.org/repos/asf/kylin/blob/89ef6775/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java index 536634d..d953d9c 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java @@ -44,10 +44,10 @@ import org.apache.kylin.engine.mr.common.BatchConstants; /** * @author yangli9 - * + * */ @SuppressWarnings({ "rawtypes", "unchecked" }) -public class MockupMapContext implements MapContext { +public class MockupMapContext implements MapContext { private Configuration hconf; @@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext { return new WrappedMapper().getMapContext(new MockupMapContext(hconf, outKV)); } - public MockupMapContext(Configuration hconf, Object[] outKV) { + public MockupMapContext(Configuration hconf, Object[] outKV){ this.hconf = hconf; this.outKV = outKV; } @@ -100,15 +100,16 @@ public class MockupMapContext implements MapContext { } @Override - public String getStatus() { + public void setStatus(String msg) { throw new NotImplementedException(); } @Override - public void setStatus(String msg) { + public String getStatus() { throw new NotImplementedException(); } + @Override public float getProgress() { throw new NotImplementedException(); } @@ -223,6 +224,7 @@ public class MockupMapContext implements MapContext { throw new NotImplementedException(); } + @Override public boolean getTaskCleanupNeeded() { throw new NotImplementedException(); } @@ -237,6 +239,7 @@ public class MockupMapContext implements MapContext { throw new NotImplementedException(); } + @Override public IntegerRanges getProfileTaskRange(boolean isMap) { throw new NotImplementedException(); } @@ -315,8 +318,4 @@ public class MockupMapContext implements MapContext { public RawComparator<?> getCombinerKeyGroupingComparator() { throw new NotImplementedException(); } - - public boolean userClassesTakesPrecedence() { - throw new NotImplementedException(); - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/89ef6775/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index 91040d4..919a903 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -60,16 +60,6 @@ <artifactId>hbase-client</artifactId> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - <scope>provided</scope> - </dependency> <!-- Env & Test --> <dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/89ef6775/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java index c0042f3..c8bff89 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java @@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.RawResource; @@ -231,7 +231,6 @@ public class CubeMigrationCLI { operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() })); } } - private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException { String projectResPath = ProjectInstance.concatResourcePath(projectName); if (!dstStore.exists(projectResPath)) @@ -448,11 +447,11 @@ public class CubeMigrationCLI { Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class); ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer); String projUUID = project.getUuid(); - Table srcAclHtable = null; - Table destAclHtable = null; + HTableInterface srcAclHtable = null; + HTableInterface destAclHtable = null; try { - srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME)); - destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME)); + srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME); + destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME); // cube acl Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId))); @@ -472,6 +471,7 @@ public class CubeMigrationCLI { destAclHtable.put(put); } } + destAclHtable.flushCommits(); } finally { IOUtils.closeQuietly(srcAclHtable); IOUtils.closeQuietly(destAclHtable); @@ -537,12 +537,13 @@ public class CubeMigrationCLI { case COPY_ACL: { String cubeId = (String) opt.params[0]; String modelId = (String) opt.params[1]; - Table destAclHtable = null; + HTableInterface destAclHtable = null; try { - destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME)); + destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME); destAclHtable.delete(new Delete(Bytes.toBytes(cubeId))); destAclHtable.delete(new Delete(Bytes.toBytes(modelId))); + destAclHtable.flushCommits(); } finally { IOUtils.closeQuietly(destAclHtable); } @@ -559,7 +560,7 @@ public class CubeMigrationCLI { } } - private static void updateMeta(KylinConfig config) { + private static void updateMeta(KylinConfig config){ String[] nodes = config.getRestServers(); for (String node : nodes) { RestClient restClient = new RestClient(node); http://git-wip-us.apache.org/repos/asf/kylin/blob/89ef6775/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java index f52fc3e..19e5db0 100644 --- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java @@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; @@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI { Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class); ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer); String projUUID = project.getUuid(); - Table aclHtable = null; + HTableInterface aclHtable = null; try { - aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl")); + aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl"); // cube acl Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId))); @@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI { aclHtable.put(put); } } + aclHtable.flushCommits(); } finally { IOUtils.closeQuietly(aclHtable); }
