This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f48a0321b314e63516ac1a0774d68b3836ba5978
Author: Zhichao Zhang <441586...@qq.com>
AuthorDate: Tue Sep 1 16:05:09 2020 +0800
KYLIN-4733 The cube size is inconsistent with the size of all segments
---
.../kylin/engine/spark/utils/BuildUtils.scala | 3 +-
.../kylin/rest/controller/CubeController.java | 45 ++++++++++++----------
.../{HBaseResponse.java => StorageResponse.java} | 4 +-
.../org/apache/kylin/rest/service/CubeService.java | 33 ++++++++--------
.../apache/kylin/rest/service/HBaseInfoUtil.java | 6 +--
webapp/app/js/controllers/cube.js | 4 +-
webapp/app/js/filters/filter.js | 4 +-
webapp/app/js/services/cubes.js | 2 +-
webapp/app/partials/cubes/cube_detail.html | 2 +-
9 files changed, 54 insertions(+), 49 deletions(-)
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
index 980c4b3..4d0db98 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
+++
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
@@ -81,7 +81,8 @@ object BuildUtils extends Logging {
@throws[IOException]
def fillCuboidInfo(cuboid: LayoutEntity, strPath: String): Unit = {
val fs = HadoopUtil.getWorkingFileSystem
- if (fs.exists(new Path(strPath))) {
+ // when cuboid.getRows == 0, it means there is no data written, so set
byte size to 0
+ if (fs.exists(new Path(strPath)) && (cuboid.getRows > 0)) {
val cs = HadoopUtil.getContentSummary(fs, new Path(strPath))
cuboid.setFileCount(cs.getFileCount)
cuboid.setByteSize(cs.getLength)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index aac0b47..a8f59f3 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -72,7 +72,7 @@ import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.GeneralResponse;
-import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.response.StorageResponse;
import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
@@ -713,15 +713,16 @@ public class CubeController extends BasicController {
}
/**
- * get Hbase Info
+ * get storage Info
*
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET
}, produces = { "application/json" })
+ @RequestMapping(value = "/{cubeName}/storage", method = {
RequestMethod.GET }, produces = {
+ "application/json" })
@ResponseBody
- public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
- List<HBaseResponse> hbase = new ArrayList<HBaseResponse>();
+ public List<StorageResponse> getStorageInfo(@PathVariable String cubeName)
{
+ List<StorageResponse> storage = new ArrayList<StorageResponse>();
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
if (null == cube) {
@@ -733,38 +734,40 @@ public class CubeController extends BasicController {
for (CubeSegment segment : segments) {
Map<String, String> addInfo = segment.getAdditionalInfo();
String tableName = segment.getStorageLocationIdentifier();
- HBaseResponse hr = null;
+ StorageResponse sr = null;
if (("" +
IStorageAware.ID_PARQUET).equals(addInfo.get("storageType"))) {
- hr = new HBaseResponse();
- hr.setStorageType("parquet");
+ sr = new StorageResponse();
+ sr.setStorageType("parquet");
+ // unit: Byte
+ sr.setTableSize(segment.getSizeKB() * 1024);
} else {
// Get info of given table.
try {
- hr = cubeService.getHTableInfo(cubeName, tableName);
+ sr = cubeService.getHTableInfo(cubeName, tableName);
} catch (IOException e) {
logger.error("Failed to calcuate size of HTable \"" +
tableName + "\".", e);
}
- if (null == hr) {
+ if (null == sr) {
logger.info("Failed to calcuate size of HTable \"" +
tableName + "\".");
- hr = new HBaseResponse();
+ sr = new StorageResponse();
}
}
- hr.setTableName(tableName);
- hr.setDateRangeStart(segment.getTSRange().start.v);
- hr.setDateRangeEnd(segment.getTSRange().end.v);
- hr.setSegmentName(segment.getName());
- hr.setSegmentStatus(segment.getStatus().toString());
- hr.setSourceCount(segment.getInputRecords());
+ sr.setTableName(tableName);
+ sr.setDateRangeStart(segment.getTSRange().start.v);
+ sr.setDateRangeEnd(segment.getTSRange().end.v);
+ sr.setSegmentName(segment.getName());
+ sr.setSegmentStatus(segment.getStatus().toString());
+ sr.setSourceCount(segment.getInputRecords());
if (segment.isOffsetCube()) {
- hr.setSourceOffsetStart((Long) segment.getSegRange().start.v);
- hr.setSourceOffsetEnd((Long) segment.getSegRange().end.v);
+ sr.setSourceOffsetStart((Long) segment.getSegRange().start.v);
+ sr.setSourceOffsetEnd((Long) segment.getSegRange().end.v);
}
- hbase.add(hr);
+ storage.add(sr);
}
- return hbase;
+ return storage;
}
/**
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/response/HBaseResponse.java
b/server-base/src/main/java/org/apache/kylin/rest/response/StorageResponse.java
similarity index 97%
rename from
server-base/src/main/java/org/apache/kylin/rest/response/HBaseResponse.java
rename to
server-base/src/main/java/org/apache/kylin/rest/response/StorageResponse.java
index cb2a108..30784b5 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/response/HBaseResponse.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/response/StorageResponse.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.response;
import java.io.Serializable;
-public class HBaseResponse implements Serializable {
+public class StorageResponse implements Serializable {
private static final long serialVersionUID = 7263557115683263492L;
private String storageType;
private String segmentName;
@@ -35,7 +35,7 @@ public class HBaseResponse implements Serializable {
private long sourceOffsetEnd;
private long sourceCount;
- public HBaseResponse() {
+ public StorageResponse() {
}
public String getStorageType() {
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index b50bdc0..d286cfd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -82,7 +82,7 @@ import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
-import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.response.StorageResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
@@ -113,7 +113,7 @@ public class CubeService extends BasicService implements
InitializingBean {
private static final Logger logger =
LoggerFactory.getLogger(CubeService.class);
- protected Cache<String, HBaseResponse> htableInfoCache =
CacheBuilder.newBuilder().build();
+ protected Cache<String, StorageResponse> storageInfoCache =
CacheBuilder.newBuilder().build();
@Autowired
@Qualifier("projectService")
@@ -530,18 +530,18 @@ public class CubeService extends BasicService implements
InitializingBean {
* table.
*
* @param tableName The table name.
- * @return The HBaseResponse object contains table size, region count. null
+ * @return The StorageResponse object contains table size, region count.
null
* if error happens
* @throws IOException Exception when HTable resource is not closed
correctly.
*/
- public HBaseResponse getHTableInfo(String cubeName, String tableName)
throws IOException {
+ public StorageResponse getHTableInfo(String cubeName, String tableName)
throws IOException {
String key = cubeName + "/" + tableName;
- HBaseResponse hr = htableInfoCache.getIfPresent(key);
- if (null != hr) {
- return hr;
+ StorageResponse sr = storageInfoCache.getIfPresent(key);
+ if (null != sr) {
+ return sr;
}
- hr = new HBaseResponse();
+ sr = new StorageResponse();
CubeInstance cube =
CubeManager.getInstance(getConfig()).getCube(cubeName);
if (cube.getStorageType() == IStorageAware.ID_HBASE ||
cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
|| cube.getStorageType() ==
IStorageAware.ID_REALTIME_AND_HBASE) {
@@ -549,17 +549,18 @@ public class CubeService extends BasicService implements
InitializingBean {
logger.debug("Loading HTable info " + cubeName + ", " +
tableName);
// use reflection to isolate NoClassDef errors when HBase is
not available
- hr = (HBaseResponse)
Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")//
- .getMethod("getHBaseInfo", new Class[] { String.class,
KylinConfig.class })//
+ sr = (StorageResponse)
+
Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")
+ .getMethod("getHBaseInfo", new Class[] { String.class,
KylinConfig.class })
.invoke(null, tableName, this.getConfig());
- hr.setStorageType("hbase");
+ sr.setStorageType("hbase");
} catch (Throwable e) {
throw new IOException(e);
}
}
- htableInfoCache.put(key, hr);
- return hr;
+ storageInfoCache.put(key, sr);
+ return sr;
}
public void updateCubeNotifyList(CubeInstance cube, List<String>
notifyList) throws IOException {
@@ -1133,7 +1134,7 @@ public class CubeService extends BasicService implements
InitializingBean {
private class HTableInfoSyncListener extends Broadcaster.Listener {
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
- htableInfoCache.invalidateAll();
+ storageInfoCache.invalidateAll();
}
@Override
@@ -1141,9 +1142,9 @@ public class CubeService extends BasicService implements
InitializingBean {
throws IOException {
String cubeName = cacheKey;
String keyPrefix = cubeName + "/";
- for (String k : htableInfoCache.asMap().keySet()) {
+ for (String k : storageInfoCache.asMap().keySet()) {
if (k.startsWith(keyPrefix))
- htableInfoCache.invalidate(k);
+ storageInfoCache.invalidate(k);
}
}
}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
index e984295..76ddbc0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
@@ -21,12 +21,12 @@ package org.apache.kylin.rest.service;
public class HBaseInfoUtil {
// @SuppressWarnings("unused") // used by reflection
-// public static HBaseResponse getHBaseInfo(String tableName, KylinConfig
config) throws IOException {
+// public static StorageResponse getHBaseInfo(String tableName, KylinConfig
config) throws IOException {
// if (!config.getStorageUrl().getScheme().equals("hbase"))
// return null;
//
// Connection conn = HBaseUnionUtil.getConnection(config, tableName);
-// HBaseResponse hr = null;
+// StorageResponse hr = null;
// long tableSize = 0;
// int regionCount = 0;
//
@@ -40,7 +40,7 @@ public class HBaseInfoUtil {
// regionCount = sizeMap.size();
//
// // Set response.
-// hr = new HBaseResponse();
+// hr = new StorageResponse();
// hr.setTableSize(tableSize);
// hr.setRegionCount(regionCount);
// return hr;
diff --git a/webapp/app/js/controllers/cube.js
b/webapp/app/js/controllers/cube.js
index 0d48dbc..60df772 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -88,9 +88,9 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope,
AccessService, Mes
});
};
- $scope.getHbaseInfo = function (cube) {
+ $scope.getStorageInfo = function (cube) {
if (!cube.hbase) {
- CubeService.getHbaseInfo({cubeId: cube.name, propValue: null,
action: null}, function (hbase) {
+ CubeService.getStorageInfo({cubeId: cube.name, propValue: null,
action: null}, function (hbase) {
cube.hbase = hbase;
TableService.get({pro:cube.model.project,
tableName:cube.model.fact_table},function(table) {
if (table && table.source_type == 1) {
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index 0f8382d..a239c17 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -87,7 +87,7 @@ KylinApp
.filter('bytes', function () {
return function (bytes, precision) {
if (bytes === 0) {
- return 'less than 1 MB';
+ return '0 KB';
}
if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) {
return '-';
@@ -97,7 +97,7 @@ KylinApp
precision = 3;
}
- var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
+ var units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'],
number = Math.floor(Math.log(bytes) / Math.log(1024));
switch(number){
case 0:
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 52cb8fe..53f490d 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -54,7 +54,7 @@ KylinApp.factory('CubeService', ['$resource', function
($resource, config) {
drop: {method: 'DELETE', params: {}, isArray: false},
save: {method: 'POST', params: {}, isArray: false},
update: {method: 'PUT', params: {}, isArray: false},
- getHbaseInfo: {method: 'GET', params: {propName: 'hbase'}, isArray: true},
+ getStorageInfo: {method: 'GET', params: {propName: 'storage'}, isArray:
true},
getCurrentCuboids: {
method: 'GET',
params: {
diff --git a/webapp/app/partials/cubes/cube_detail.html
b/webapp/app/partials/cubes/cube_detail.html
index 7cb0351..0076069 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -39,7 +39,7 @@
</li>
<li class="{{cube.visiblePage=='hbase'? 'active':''}}"
ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission('cube'
,cube, permissions.ADMINISTRATION.mask) && !newAccess">
- <a href=""
ng-click="cube.visiblePage='hbase';getHbaseInfo(cube)">Storage</a>
+ <a href=""
ng-click="cube.visiblePage='hbase';getStorageInfo(cube)">Storage</a>
</li>
<li class="{{cube.visiblePage=='planner'? 'active':''}}"
ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission('cube', cube,
permissions.ADMINISTRATION.mask) && !newAccess) && isShowCubeplanner &&
cube.detail.engine_type != 6">
<a href=""
ng-click="cube.visiblePage='planner';getCubePlanner(cube);">Planner</a>
