Repository: kylin Updated Branches: refs/heads/master 5edffa51d -> 99f1dd9d2
KYLIN-1793 Fix exit code issue in beeline mode Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99f1dd9d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99f1dd9d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99f1dd9d Branch: refs/heads/master Commit: 99f1dd9d2460566748db2167e69a6a8a7689271d Parents: 5edffa5 Author: lidongsjtu <lid...@apache.org> Authored: Mon Nov 28 08:35:05 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Mon Nov 28 08:46:35 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 2 +- .../kylin/common/util/HiveCmdBuilder.java | 114 +++++++++++++++++++ .../kylin/common/util/HiveCmdBuilderTest.java | 80 +++++++++++++ .../source/hive/CreateFlatHiveTableStep.java | 1 + .../kylin/source/hive/HiveCmdBuilder.java | 113 ------------------ .../apache/kylin/source/hive/HiveMRInput.java | 1 + .../kylin/source/hive/HiveCmdBuilderTest.java | 83 -------------- .../storage/hbase/steps/DeprecatedGCStep.java | 2 +- .../storage/hbase/util/HiveCmdBuilder.java | 110 ------------------ .../storage/hbase/util/StorageCleanupJob.java | 3 +- .../apache/kylin/tool/StorageCleanupJob.java | 2 +- 11 files changed, 201 insertions(+), 310 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index 79a3b3b..23b3670 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -46,7 +46,7 @@ import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.source.hive.HiveClientFactory; -import org.apache.kylin.source.hive.HiveCmdBuilder; +import org.apache.kylin.common.util.HiveCmdBuilder; import org.apache.kylin.source.hive.IHiveClient; import org.apache.kylin.source.kafka.TimedJsonStreamParser; import org.apache.maven.model.Model; http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java new file mode 100644 index 0000000..5942ba9 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.kylin.common.KylinConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class HiveCmdBuilder { + private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class); + + public enum HiveClientMode { + CLI, BEELINE + } + + private HiveClientMode clientMode; + private KylinConfig kylinConfig; + final private ArrayList<String> statements = Lists.newArrayList(); + + public HiveCmdBuilder() { + kylinConfig = KylinConfig.getInstanceFromEnv(); + clientMode = HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase()); + } + + public String build() { + StringBuffer buf = new StringBuffer(); + + switch (clientMode) { + case CLI: + buf.append("hive -e \""); + for (String statement : statements) { + buf.append(statement).append("\n"); + } + buf.append("\""); + break; + case BEELINE: + BufferedWriter bw = null; + try { + File tmpHql = File.createTempFile("beeline_", ".hql"); + bw = new BufferedWriter(new FileWriter(tmpHql)); + for (String statement : statements) { + bw.write(statement); + bw.newLine(); + } + buf.append("beeline "); + buf.append(kylinConfig.getHiveBeelineParams()); + buf.append(" -f "); + buf.append(tmpHql.getAbsolutePath()); + buf.append(";ret_code=$?;rm -f "); + buf.append(tmpHql.getAbsolutePath()); + buf.append(";exit $ret_code"); + + if (logger.isDebugEnabled()) { + logger.debug("The SQL to execute in beeline: \n" + FileUtils.readFileToString(tmpHql, Charset.defaultCharset())); + } + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + IOUtils.closeQuietly(bw); + } + break; + default: + throw new RuntimeException("Hive client cannot be recognized: " + clientMode); + } + + return buf.toString(); + } + + public void reset() { + statements.clear(); + } + + public void addStatement(String statement) { + statements.add(statement); + } + + public void addStatements(String[] stats) { + for (String s : stats) { + statements.add(s); + } + } + + @Override + public String toString() { + return build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java new file mode 100644 index 0000000..56a64c8 --- /dev/null +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class HiveCmdBuilderTest { + + @Before + public void setup() { + System.setProperty("KYLIN_CONF", LocalFileMetadataTestCase.LOCALMETA_TEST_DATA); + } + + @After + public void after() throws Exception { + System.clearProperty("kylin.source.hive.client"); + System.clearProperty("kylin.source.hive.beeline-params"); + } + + @Test + public void testHiveCLI() { + System.setProperty("kylin.source.hive.client", "cli"); + + HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); + hiveCmdBuilder.addStatement("USE default;"); + hiveCmdBuilder.addStatement("DROP TABLE test;"); + hiveCmdBuilder.addStatement("SHOW\n TABLES;"); + + assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\"", hiveCmdBuilder.build()); + } + + @Test + public void testBeeline() throws IOException { + String lineSeparator = java.security.AccessController.doPrivileged(new sun.security.action.GetPropertyAction("line.separator")); + System.setProperty("kylin.source.hive.client", "beeline"); + System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url"); + + HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); + hiveCmdBuilder.addStatement("USE default;"); + hiveCmdBuilder.addStatement("DROP TABLE test;"); + hiveCmdBuilder.addStatement("SHOW\n TABLES;"); + + String cmd = hiveCmdBuilder.build(); + assertTrue(cmd.startsWith("beeline -u jdbc_url -f")); + + String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim(); + hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length()); + + String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset()); + assertEquals("USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW\n TABLES;" + lineSeparator, hqlStatement); + + FileUtils.forceDelete(new File(hqlFile)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java index 025fd94..f0e5703 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BufferedLogger; +import org.apache.kylin.common.util.HiveCmdBuilder; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java deleted file mode 100644 index 4c8aeea..0000000 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.source.hive; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.kylin.common.KylinConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -public class HiveCmdBuilder { - private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class); - - public enum HiveClientMode { - CLI, BEELINE - } - - private HiveClientMode clientMode; - private KylinConfig kylinConfig; - final private ArrayList<String> statements = Lists.newArrayList(); - - public HiveCmdBuilder() { - kylinConfig = KylinConfig.getInstanceFromEnv(); - clientMode = HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase()); - } - - public String build() { - StringBuffer buf = new StringBuffer(); - - switch (clientMode) { - case CLI: - buf.append("hive -e \""); - for (String statement : statements) { - buf.append(statement).append("\n"); - } - buf.append("\""); - break; - case BEELINE: - BufferedWriter bw = null; - try { - File tmpHql = File.createTempFile("beeline_", ".hql"); - bw = new BufferedWriter(new FileWriter(tmpHql)); - for (String statement : statements) { - bw.write(statement); - bw.newLine(); - } - buf.append("beeline "); - buf.append(kylinConfig.getHiveBeelineParams()); - buf.append(" -f "); - buf.append(tmpHql.getAbsolutePath()); - buf.append(";rm -f "); - buf.append(tmpHql.getAbsolutePath()); - - if (logger.isDebugEnabled()) { - logger.debug("The SQL to execute in beeline: \n" + FileUtils.readFileToString(tmpHql, Charset.defaultCharset())); - } - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - IOUtils.closeQuietly(bw); - } - break; - default: - throw new RuntimeException("Hive client cannot be recognized: " + clientMode); - } - - return buf.toString(); - } - - public void reset() { - statements.clear(); - } - - public void addStatement(String statement) { - statements.add(statement); - } - - public void addStatements(String[] stats) { - for (String s : stats) { - statements.add(s); - } - } - - @Override - public String toString() { - return build(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 6eb1a28..aa603d7 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -30,6 +30,7 @@ import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BufferedLogger; +import org.apache.kylin.common.util.HiveCmdBuilder; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java ---------------------------------------------------------------------- diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java deleted file mode 100644 index 0eb02b4..0000000 --- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.source.hive; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; - -import org.apache.commons.io.FileUtils; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class HiveCmdBuilderTest { - - @Before - public void setup() { - System.setProperty("KYLIN_CONF", LocalFileMetadataTestCase.LOCALMETA_TEST_DATA); - } - - @After - public void after() throws Exception { - System.clearProperty("kylin.source.hive.client"); - System.clearProperty("kylin.source.hive.beeline-params"); - } - - @Test - public void testHiveCLI() { - System.setProperty("kylin.source.hive.client", "cli"); - - HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - hiveCmdBuilder.addStatement("USE default;"); - hiveCmdBuilder.addStatement("DROP TABLE test;"); - hiveCmdBuilder.addStatement("SHOW\n TABLES;"); - - assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\"", hiveCmdBuilder.build()); - } - - @Test - public void testBeeline() throws IOException { - String lineSeparator = java.security.AccessController.doPrivileged(new sun.security.action.GetPropertyAction("line.separator")); - System.setProperty("kylin.source.hive.client", "beeline"); - System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url"); - - HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - hiveCmdBuilder.addStatement("USE default;"); - hiveCmdBuilder.addStatement("DROP TABLE test;"); - hiveCmdBuilder.addStatement("SHOW\n TABLES;"); - - String cmd = hiveCmdBuilder.build(); - assertTrue(cmd.startsWith("beeline -u jdbc_url -f")); - - String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim(); - if (hqlFile.endsWith(";")) { - hqlFile = hqlFile.substring(0, hqlFile.length() - 1); - } - - String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset()); - assertEquals("USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW\n TABLES;" + lineSeparator, hqlStatement); - - FileUtils.forceDelete(new File(hqlFile)); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java index 21f1c00..46a828e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.HiveCmdBuilder; import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; @@ -39,7 +40,6 @@ import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.storage.hbase.HBaseConnection; -import org.apache.kylin.storage.hbase.util.HiveCmdBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java deleted file mode 100644 index 9252254..0000000 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.hbase.util; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.io.IOUtils; -import org.apache.kylin.common.KylinConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -/** - * Created by dongli on 2/29/16. - */ -@Deprecated -public class HiveCmdBuilder { - private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class); - - public enum HiveClientMode { - CLI, BEELINE - } - - private HiveClientMode clientMode; - private KylinConfig kylinConfig; - final private ArrayList<String> statements = Lists.newArrayList(); - - public HiveCmdBuilder() { - kylinConfig = KylinConfig.getInstanceFromEnv(); - clientMode = HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase()); - } - - public String build() { - StringBuffer buf = new StringBuffer(); - - switch (clientMode) { - case CLI: - buf.append("hive -e \""); - for (String statement : statements) { - buf.append(statement).append("\n"); - } - buf.append("\""); - break; - case BEELINE: - BufferedWriter bw = null; - try { - File tmpHql = File.createTempFile("beeline_", ".hql"); - StringBuffer hqlBuf = new StringBuffer(); - bw = new BufferedWriter(new FileWriter(tmpHql)); - for (String statement : statements) { - bw.write(statement); - bw.newLine(); - - hqlBuf.append(statement).append("\n"); - } - buf.append("beeline "); - buf.append(kylinConfig.getHiveBeelineParams()); - buf.append(" -f "); - buf.append(tmpHql.getAbsolutePath()); - buf.append(";rm -f "); - buf.append(tmpHql.getAbsolutePath()); - - logger.info("The statements to execute in beeline: \n" + hqlBuf); - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - IOUtils.closeQuietly(bw); - } - break; - default: - throw new RuntimeException("Hive client cannot be recognized: " + clientMode); - } - - return buf.toString(); - } - - public void reset() { - statements.clear(); - } - - public void addStatement(String statement) { - statements.add(statement); - } - - @Override - public String toString() { - return build(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java index 08471a3..90080b6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.CliCommandExecutor; +import org.apache.kylin.common.util.HiveCmdBuilder; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeInstance; @@ -83,7 +84,7 @@ public class StorageCleanupJob extends AbstractApplication { String host = desc.getValue(IRealizationConstants.HTableTag); if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) { //only take care htables that belongs to self, and created more than 2 days - allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString()); + allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/99f1dd9d/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java index b51a850..b4accac 100644 --- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java +++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java @@ -57,7 +57,7 @@ import org.apache.kylin.job.execution.ExecutableManager; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.source.hive.HiveClientFactory; -import org.apache.kylin.source.hive.HiveCmdBuilder; +import org.apache.kylin.common.util.HiveCmdBuilder; import org.apache.kylin.source.hive.IHiveClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory;
