KYLIN-3172 No such file or directory error with CreateLookupHiveViewMaterializationStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35979ff0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35979ff0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35979ff0 Branch: refs/heads/master Commit: 35979ff0875d06cb4b1cc4b151efe6af37d1318d Parents: 195fdef Author: yiming.xu <100650...@qq.com> Authored: Wed Jan 17 18:37:08 2018 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Jan 26 22:54:58 2018 +0800 ---------------------------------------------------------------------- .../kylin/common/util/HiveCmdBuilder.java | 45 ++++++--------- .../kylin/common/util/HiveCmdBuilderTest.java | 33 +++++++---- .../apache/kylin/source/hive/HiveMRInput.java | 36 ++++++------ .../kylin/source/hive/HiveMRInputTest.java | 59 ++++++++++++++++++++ 4 files changed, 115 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index add53db..c33a7fb 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -18,11 +18,7 @@ package org.apache.kylin.common.util; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -30,8 +26,6 @@ import java.util.Map; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.slf4j.Logger; @@ -42,9 +36,10 @@ import org.w3c.dom.NodeList; import com.google.common.collect.Lists; public class HiveCmdBuilder { - private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class); + public static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class); public static final String HIVE_CONF_FILENAME = "kylin_hive_conf"; + static final String CREATE_HQL_TMP_FILE_TEMPLATE = "cat >%s<<EOL\n%sEOL"; public enum HiveClientMode { CLI, BEELINE @@ -68,10 +63,11 @@ public class HiveCmdBuilder { beelineShell = kylinConfig.getSparkSqlBeelineShell(); beelineParams = kylinConfig.getSparkSqlBeelineParams(); if (StringUtils.isBlank(beelineShell)) { - throw new IllegalStateException("Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties"); + throw new IllegalStateException( + "Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties"); } } - + StringBuffer buf = new StringBuffer(); switch (clientMode) { @@ -84,37 +80,28 @@ public class HiveCmdBuilder { buf.append(parseProps()); break; case BEELINE: - BufferedWriter bw = null; - File tmpHql = null; + String tmpHqlPath = null; + StringBuilder hql = new StringBuilder(); try { - tmpHql = File.createTempFile("beeline_", ".hql"); - bw = new BufferedWriter(new FileWriter(tmpHql)); + tmpHqlPath = "/tmp/" + System.currentTimeMillis() + ".hql"; for (String statement : statements) { - bw.write(statement); - bw.newLine(); + hql.append(statement); + hql.append("\n"); } + String createFileCmd = String.format(CREATE_HQL_TMP_FILE_TEMPLATE, tmpHqlPath, hql); + buf.append(createFileCmd); + buf.append("\n"); buf.append(beelineShell); buf.append(" "); buf.append(beelineParams); buf.append(parseProps()); buf.append(" -f "); - buf.append(tmpHql.getAbsolutePath()); + buf.append(tmpHqlPath); buf.append(";ret_code=$?;rm -f "); - buf.append(tmpHql.getAbsolutePath()); + buf.append(tmpHqlPath); buf.append(";exit $ret_code"); - - } catch (IOException e) { - throw new RuntimeException(e); } finally { - IOUtils.closeQuietly(bw); - - if (tmpHql != null && logger.isDebugEnabled()) { - String hql = null; - try { - hql = FileUtils.readFileToString(tmpHql, Charset.defaultCharset()); - } catch (IOException e) { - // ignore - } + if (tmpHqlPath != null && logger.isDebugEnabled()) { logger.debug("The SQL to execute in beeline: \n" + hql); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java index 4262722..ecc8961 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -45,7 +45,7 @@ public class HiveCmdBuilderTest { System.clearProperty("kylin.source.hive.client"); System.clearProperty("kylin.source.hive.beeline-shell"); System.clearProperty("kylin.source.hive.beeline-params"); - + System.clearProperty("kylin.source.hive.enable-sparksql-for-table-ops"); System.clearProperty("kylin.source.hive.sparksql-beeline-shell"); System.clearProperty("kylin.source.hive.sparksql-beeline-params"); @@ -65,12 +65,15 @@ public class HiveCmdBuilderTest { hiveCmdBuilder.addStatement("SHOW\n TABLES;"); hiveCmdBuilder.setHiveConfProps(hiveProps); hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite); - assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); + assertEquals( + "hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", + hiveCmdBuilder.build()); } @Test public void testBeeline() throws IOException { - String lineSeparator = java.security.AccessController.doPrivileged(new sun.security.action.GetPropertyAction("line.separator")); + String lineSeparator = java.security.AccessController + .doPrivileged(new sun.security.action.GetPropertyAction("line.separator")); System.setProperty("kylin.source.hive.client", "beeline"); System.setProperty("kylin.source.hive.beeline-shell", "/spark-client/bin/beeline"); System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url"); @@ -78,17 +81,19 @@ public class HiveCmdBuilderTest { HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement("USE default;"); hiveCmdBuilder.addStatement("DROP TABLE test;"); - hiveCmdBuilder.addStatement("SHOW\n TABLES;"); + hiveCmdBuilder.addStatement("SHOW TABLES;"); String cmd = hiveCmdBuilder.build(); - assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url")); - String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim(); hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length()); - + String createFileCmd = cmd.substring(0, cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 3); + CliCommandExecutor cliCommandExecutor = new CliCommandExecutor(); + Pair<Integer, String> execute = cliCommandExecutor.execute(createFileCmd); String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset()); - assertEquals("USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW\n TABLES;" + lineSeparator, hqlStatement); - + assertEquals( + "USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW TABLES;" + lineSeparator, + hqlStatement); + assertBeelineCmd(cmd); FileUtils.forceDelete(new File(hqlFile)); } @@ -101,9 +106,13 @@ public class HiveCmdBuilderTest { HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement("USE default;"); hiveCmdBuilder.addStatement("DROP TABLE test;"); - hiveCmdBuilder.addStatement("SHOW\n TABLES;"); - + hiveCmdBuilder.addStatement("SHOW TABLES;"); String cmd = hiveCmdBuilder.build(); - assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url")); + assertBeelineCmd(cmd); + } + + private void assertBeelineCmd(String cmd) { + String beelineCmd = cmd.substring(cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 4); + assertTrue(beelineCmd.startsWith("/spark-client/bin/beeline -u jdbc_url")); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 6f42961..31a99cd 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -177,7 +177,25 @@ public class HiveMRInput implements IMRInput { } protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) { - return JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId()); + + String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId()); + // Create work dir to avoid hive create it, + // the difference is that the owners are different. + checkAndCreateWorkDir(jobWorkingDir); + return jobWorkingDir; + } + + private void checkAndCreateWorkDir(String jobWorkingDir) { + try { + Path path = new Path(jobWorkingDir); + FileSystem fileSystem = HadoopUtil.getFileSystem(path); + if (!fileSystem.exists(path)) { + logger.info("Create jobWorkDir : " + jobWorkingDir); + fileSystem.mkdirs(path); + } + } catch (IOException e) { + logger.error("Could not create lookUp table dir : " + jobWorkingDir); + } } private AbstractExecutable createRedistributeFlatHiveTableStep(String hiveInitStatements, String cubeName) { @@ -210,9 +228,6 @@ public class HiveMRInput implements IMRInput { if (lookupViewsTables.size() == 0) { return null; } - // Create work dir to avoid hive create it, - // the difference is that the owners are different. - checkAndCreateWorkDir(jobWorkingDir); HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride()); @@ -242,19 +257,6 @@ public class HiveMRInput implements IMRInput { return step; } - private void checkAndCreateWorkDir(String jobWorkingDir) { - try { - Path path = new Path(jobWorkingDir); - FileSystem fileSystem = HadoopUtil.getFileSystem(path); - if (!fileSystem.exists(path)) { - logger.info("Create jobWorkDir : " + jobWorkingDir); - fileSystem.mkdirs(path); - } - } catch (IOException e) { - logger.error("Could not create lookUp table dir : " + jobWorkingDir); - } - } - private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir, String cubeName) { //from hive to hive http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java ---------------------------------------------------------------------- diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java new file mode 100644 index 0000000..6d0737d --- /dev/null +++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.source.hive; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.UUID; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.job.execution.DefaultChainedExecutable; +import org.junit.Assert; +import org.junit.Test; + +public class HiveMRInputTest { + + @Test + public void TestGetJobWorkingDir() throws IOException { + FileSystem fileSystem = FileSystem.get(new Configuration()); + Path jobWorkDirPath = null; + try { + KylinConfig kylinConfig = mock(KylinConfig.class); + KylinConfig.setKylinConfigThreadLocal(kylinConfig); + when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/"); + + DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class); + defaultChainedExecutable.setId(UUID.randomUUID().toString()); + + HiveMRInput.BatchCubingInputSide batchCubingInputSide = new HiveMRInput.BatchCubingInputSide(null); + String jobWorkingDir = batchCubingInputSide.getJobWorkingDir(defaultChainedExecutable); + jobWorkDirPath = new Path(jobWorkingDir); + Assert.assertTrue(fileSystem.exists(jobWorkDirPath)); + } finally { + if (jobWorkDirPath != null) + fileSystem.deleteOnExit(jobWorkDirPath); + } + } + +} \ No newline at end of file
