KYLIN-1758 add the hive set statements
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1d17e21 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1d17e21 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1d17e21 Branch: refs/heads/v1.5.2.1-release-rc1 Commit: a1d17e215561089bb5958b84e876a74fc60f092f Parents: ef1d13f Author: shaofengshi <shaofeng...@apache.org> Authored: Sat Jun 4 16:22:38 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sat Jun 4 16:22:38 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/job/JoinedFlatTable.java | 56 +++++++++++--------- .../apache/kylin/source/hive/HiveMRInput.java | 14 +++++ 2 files changed, 44 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a1d17e21/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 5886325..f13a48c 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -55,6 +55,36 @@ public class JoinedFlatTable { return storageDfsDir + "/" + intermediateTableDesc.getTableName(); } + public static String generateHiveSetStatements(JobEngineConfig engineConfig) throws IOException { + StringBuilder buffer = new StringBuilder(); + File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath()); + + if (hadoopPropertiesFile.exists()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder; + Document doc; + try { + builder = factory.newDocumentBuilder(); + doc = builder.parse(hadoopPropertiesFile); + NodeList nl = doc.getElementsByTagName("property"); + for (int i = 0; i < nl.getLength(); i++) { + String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); + String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue(); + if (!name.equals("tmpjars")) { + buffer.append("SET " + name + "=" + value + ";\n"); + } + } + + } catch (ParserConfigurationException e) { + throw new IOException(e); + } catch (SAXException e) { + throw new IOException(e); + } + } + + return buffer.toString(); + } + public static String generateCreateTableStatement(IJoinedFlatTableDesc intermediateTableDesc, String storageDfsDir) { StringBuilder ddl = new StringBuilder(); @@ -86,32 +116,6 @@ public class JoinedFlatTable { public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) throws IOException { StringBuilder sql = new StringBuilder(); - - File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath()); - - if (hadoopPropertiesFile.exists()) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder; - Document doc; - try { - builder = factory.newDocumentBuilder(); - doc = builder.parse(hadoopPropertiesFile); - NodeList nl = doc.getElementsByTagName("property"); - for (int i = 0; i < nl.getLength(); i++) { - String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); - String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue(); - if (name.equals("tmpjars") == false) { - sql.append("SET " + name + "=" + value + ";").append("\n"); - } - } - - } catch (ParserConfigurationException e) { - throw new IOException(e); - } catch (SAXException e) { - throw new IOException(e); - } - } - sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc) + ";").append("\n"); return sql.toString(); http://git-wip-us.apache.org/repos/asf/kylin/blob/a1d17e21/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 16642fa..4026006 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -117,6 +117,12 @@ public class HiveMRInput implements IMRInput { } public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId) { + String setHql = ""; + try { + setHql = JoinedFlatTable.generateHiveSetStatements(conf); + } catch (IOException e) { + throw new RuntimeException("Failed to generate hive set statements for createFlatHiveTableStep", e); + } final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";"; final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc); @@ -134,6 +140,7 @@ public class HiveMRInput implements IMRInput { hiveCmdBuilder.addStatement(useDatabaseHql); hiveCmdBuilder.addStatement(dropTableHql); hiveCmdBuilder.addStatement(createTableHql); + hiveCmdBuilder.addStatement(setHql); hiveCmdBuilder.addStatement(insertDataHqls); step.setCmd(hiveCmdBuilder.build()); @@ -164,8 +171,15 @@ public class HiveMRInput implements IMRInput { if(lookupViewsTables.size() == 0) { return null; } + String setHql = ""; + try { + setHql = JoinedFlatTable.generateHiveSetStatements(conf); + } catch (IOException e) { + throw new RuntimeException("Failed to generate hive set statements for createFlatHiveTableStep", e); + } final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";"; hiveCmdBuilder.addStatement(useDatabaseHql); + hiveCmdBuilder.addStatement(setHql); for(TableDesc lookUpTableDesc : lookupViewsTables) { if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) { StringBuilder createIntermediateTableHql = new StringBuilder();
