Minor, fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca8428d3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca8428d3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca8428d3 Branch: refs/heads/sync Commit: ca8428d388cb3b0716efc92fb5cec729d636204f Parents: b4daa8a Author: Yifan Zhang <event.dim...@gmail.com> Authored: Mon Jan 1 16:33:26 2018 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Jan 26 17:24:08 2018 +0800 ---------------------------------------------------------------------- .../kylin/source/hive/BeelineHiveClient.java | 5 +-- .../apache/kylin/source/hive/CLIHiveClient.java | 39 +++++++------------- .../kylin/source/hive/HiveMetadataExplorer.java | 29 ++++++++++----- .../apache/kylin/source/hive/IHiveClient.java | 6 +-- .../apache/kylin/source/jdbc/JdbcExplorer.java | 9 ++++- 5 files changed, 43 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java index ee693c5..314402c 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java @@ -28,7 +28,6 @@ import java.sql.Statement; import java.util.List; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.kylin.common.util.DBUtils; import com.google.common.base.Preconditions; @@ -117,12 +116,12 @@ public class BeelineHiveClient implements IHiveClient { } @Override - public void executeHQL(String hql) throws CommandNeedRetryException, IOException { + public void executeHQL(String hql) throws IOException { throw new UnsupportedOperationException(); } @Override - public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException { + public void executeHQL(String[] hqls) throws IOException { throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java index e8a93bd..bc9f17e 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java @@ -22,17 +22,15 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; -import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.HiveCmdBuilder; +import org.apache.kylin.common.util.Pair; import com.google.common.collect.Lists; @@ -43,7 +41,6 @@ import com.google.common.collect.Lists; */ public class CLIHiveClient implements IHiveClient { protected HiveConf hiveConf = null; - protected Driver driver = null; protected HiveMetaStoreClient metaStoreClient = null; public CLIHiveClient() { @@ -52,22 +49,25 @@ public class CLIHiveClient implements IHiveClient { /** * only used by Deploy Util + * @throws IOException */ @Override - public void executeHQL(String hql) throws CommandNeedRetryException, IOException { - CommandProcessorResponse response = getDriver().run(hql); - int retCode = response.getResponseCode(); - if (retCode != 0) { - String err = response.getErrorMessage(); - throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err); + public void executeHQL(String hql) throws IOException { + final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); + hiveCmdBuilder.addStatement(hql); + Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor() + .execute(hiveCmdBuilder.toString()); + if (response.getFirst() != 0) { + throw new IllegalArgumentException("Failed to execute hql [" + hql + "], error message is: " + response.getSecond()); } + } /** * only used by Deploy Util */ @Override - public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException { + public void executeHQL(String[] hqls) throws IOException { for (String sql : hqls) executeHQL(sql); } @@ -159,17 +159,4 @@ public class CLIHiveClient implements IHiveClient { } return result; } - - /** - * Get the hive ql driver to execute ddl or dml - * @return - */ - private Driver getDriver() { - if (driver == null) { - driver = new Driver(hiveConf); - SessionState.start(new CliSessionState(hiveConf)); - } - - return driver; - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java index d80c546..cb3eb02 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java @@ -32,11 +32,15 @@ import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TableExtDesc; import org.apache.kylin.source.ISampleDataDeployer; import org.apache.kylin.source.ISourceMetadataExplorer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDataDeployer { + private static final Logger logger = LoggerFactory.getLogger(HiveClientFactory.class); + IHiveClient hiveClient = HiveClientFactory.getHiveClient(); - + @Override public List<String> listDatabases() throws Exception { return hiveClient.getHiveDbNames(); @@ -90,7 +94,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat tableExtDesc.setUuid(UUID.randomUUID().toString()); tableExtDesc.setLastModified(0); tableExtDesc.init(prj); - + tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation); tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner); tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime)); @@ -114,10 +118,10 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat hiveClient.executeHQL(generateCreateSchemaSql(database)); } - private String generateCreateSchemaSql(String schemaName){ + private String generateCreateSchemaSql(String schemaName) { return String.format("CREATE DATABASE IF NOT EXISTS %s", schemaName); } - + @Override public void createSampleTable(TableDesc table) throws Exception { hiveClient.executeHQL(generateCreateTableSql(table)); @@ -146,7 +150,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat return new String[] { dropsql, dropsql2, ddl.toString() }; } - + @Override public void loadSampleData(String tableName, String tmpDataDir) throws Exception { hiveClient.executeHQL(generateLoadDataSql(tableName, tmpDataDir)); @@ -155,12 +159,12 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat private String generateLoadDataSql(String tableName, String tableFileDir) { return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName; } - + @Override public void createWrapperView(String origTableName, String viewName) throws Exception { hiveClient.executeHQL(generateCreateViewSql(viewName, origTableName)); } - + private String[] generateCreateViewSql(String viewName, String tableName) { String dropView = "DROP VIEW IF EXISTS " + viewName; @@ -170,7 +174,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat return new String[] { dropView, dropTable, createSql }; } - + private static String getHiveDataType(String javaDataType) { String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType; hiveDataType = javaDataType.toLowerCase().startsWith("integer") ? "int" : hiveDataType; @@ -192,16 +196,21 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat String evalViewSql = "CREATE VIEW " + tmpDatabase + "." + tmpView + " as " + query; try { - hiveClient.executeHQL(new String[] { dropViewSql, evalViewSql }); + logger.debug("Removing duplicate view {}", tmpView); + hiveClient.executeHQL(dropViewSql); + logger.debug("Creating view {} for query: {}", tmpView, query); + hiveClient.executeHQL(evalViewSql); + logger.debug("Evaluating query columns' metadata"); HiveTableMeta hiveTableMeta = hiveClient.getHiveTableMeta(tmpDatabase, tmpView); return extractColumnFromMeta(hiveTableMeta); } catch (Exception e) { throw new RuntimeException("Cannot evalutate metadata of query: " + query, e); } finally { try { + logger.debug("Cleaning up."); hiveClient.executeHQL(dropViewSql); } catch (Exception e) { - throw new RuntimeException("Cannot temp view of query: " + query, e); + logger.warn("Cannot drop temp view of query: {}", query, e); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java index ca5312d..ec2bf7f 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java @@ -21,13 +21,11 @@ package org.apache.kylin.source.hive; import java.io.IOException; import java.util.List; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; - public interface IHiveClient { - void executeHQL(String hql) throws CommandNeedRetryException, IOException; + void executeHQL(String hql) throws IOException; - void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException; + void executeHQL(String[] hqls) throws IOException; HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception; http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java index 81e42bb..d96a68d 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java @@ -243,7 +243,11 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye String evalViewSql = "CREATE VIEW " + tmpView + " as " + query; try { - executeSQL(new String[] { dropViewSql, evalViewSql }); + logger.debug("Removing duplicate view {}", tmpView); + executeSQL(dropViewSql); + logger.debug("Creating view {} for query: {}", tmpView, query); + executeSQL(evalViewSql); + logger.debug("Evaluating query columns' metadata"); Connection con = SqlUtil.getConnection(dbconf); DatabaseMetaData dbmd = con.getMetaData(); ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null); @@ -255,9 +259,10 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye throw new RuntimeException("Cannot evalutate metadata of query: " + query, e); } finally { try { + logger.debug("Cleaning up."); executeSQL(dropViewSql); } catch (Exception e) { - throw new RuntimeException("Cannot temp view of query: " + query, e); + logger.warn("Cannot drop temp view of query: {}", query, e); } } }
