This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 2848bef KYLIN-4526 Enhance get the hive table rows
2848bef is described below
commit 2848bef54c9b56068d8800e2fdf44851d4b150d3
Author: Guangxu Cheng <gxch...@apache.org>
AuthorDate: Wed May 27 12:09:39 2020 +0800
KYLIN-4526 Enhance get the hive table rows
---
.../kylin/source/hive/BeelineHiveClient.java | 30 +++++++++++++++++++++-
.../apache/kylin/source/hive/CLIHiveClient.java | 19 +++++++++++++-
2 files changed, 47 insertions(+), 2 deletions(-)
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 4fcb73a..e91552d 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -32,6 +32,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
@@ -131,7 +132,34 @@ public class BeelineHiveClient implements IHiveClient {
@Override
public long getHiveTableRows(String database, String tableName) throws
Exception {
- return getHiveTableMeta(database, tableName).rowNum;
+ ResultSet resultSet = null;
+ long count = 0;
+ try {
+ HiveTableMetaBuilder builder = new HiveTableMetaBuilder();
+ String exe = "use ";
+ stmt.execute(exe.concat(database));
+ String des = "describe formatted ";
+ resultSet = stmt.executeQuery(des.concat(tableName));
+ extractHiveTableMeta(resultSet, builder);
+ count = builder.createHiveTableMeta().rowNum;
+ if (count <= 0) {
+ String querySQL = "select count(*) from ".concat(database +
"." + tableName);
+ List<Object[]> datas = null;
+ try {
+ datas = getHiveResult(querySQL);
+ if (Objects.nonNull(datas) && !datas.isEmpty()) {
+ count = Integer.parseInt(datas.get(0)[0] + "");
+ } else {
+ throw new IOException("execute get hive table rows
result fail : " + querySQL);
+ }
+ } catch (Exception e) {
+ throw new IOException("execute get hive table rows result
fail : " + querySQL, e);
+ }
+ }
+ } finally {
+ DBUtils.closeQuietly(resultSet);
+ }
+ return count;
}
@Override
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index 7421770..62d995e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
/**
* Hive meta API client for Kylin
@@ -126,8 +127,24 @@ public class CLIHiveClient implements IHiveClient {
@Override
public long getHiveTableRows(String database, String tableName) throws
Exception {
+ long count = 0;
Table table = getMetaStoreClient().getTable(database, tableName);
- return getBasicStatForTable(new
org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.ROW_COUNT);
+ count = getBasicStatForTable(new
org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.ROW_COUNT);
+ if (count <= 0) {
+ String querySQL = "select count(*) from ".concat(database + "." +
tableName);
+ List<Object[]> datas = null;
+ try {
+ datas = getHiveResult(querySQL);
+ if (Objects.nonNull(datas) && !datas.isEmpty()) {
+ count = Integer.parseInt(datas.get(0)[0] + "");
+ } else {
+ throw new IOException("execute get hive table rows result
fail : " + querySQL);
+ }
+ } catch (Exception e) {
+ throw new IOException("execute get hive table rows result fail
: " + querySQL, e);
+ }
+ }
+ return count;
}
@Override
