[kylin] branch 2.6.x updated: KYLIN-3994: StorageCleanupJob may delete cube id data of new built segment because of cube cache in CubeManager (#633)

[nic]

This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/2.6.x by this push:
     new 0c1dd10  KYLIN-3994: StorageCleanupJob may delete cube id data of new 
built segment because of cube cache in CubeManager (#633)
0c1dd10 is described below

commit 0c1dd10fd98062df65b7b55ba46cba1eda0ff1ff
Author: Liu Shaohui <lshmo...@gmail.com>
AuthorDate: Fri May 31 18:18:19 2019 +0800

    KYLIN-3994: StorageCleanupJob may delete cube id data of new built segment 
because of cube cache in CubeManager (#633)
    
    * KYLIN-3994: StorageCleanupJob may delete cube id data of new built 
segment because of cube cache in CubeManager
---
 .../src/main/java/org/apache/kylin/cube/CubeManager.java | 16 ++++++++++++++++
 .../org/apache/kylin/rest/job/MetadataCleanupJob.java    |  2 +-
 .../apache/kylin/rest/job/StorageCleanJobHbaseUtil.java  |  2 +-
 .../org/apache/kylin/rest/job/StorageCleanupJob.java     |  2 +-
 4 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index bd9832f..db9c095 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -182,12 +182,28 @@ public class CubeManager implements IRealizationProvider {
         }
     }
 
+    /**
+     * List all cubes from cache. Note the metadata may be out of date
+     * @return
+     */
     public List<CubeInstance> listAllCubes() {
         try (AutoLock lock = cubeMapLock.lockForRead()) {
             return new ArrayList<CubeInstance>(cubeMap.values());
         }
     }
 
+    /**
+     * Reload the cubes from database and list all cubes
+     * @return
+     * @throws IOException
+     */
+    public List<CubeInstance> reloadAndListAllCubes() throws IOException {
+        try (AutoLock lock = cubeMapLock.lockForWrite()) {
+            crud.reloadAll();
+        }
+        return listAllCubes();
+    }
+
     public CubeInstance getCube(String cubeName) {
         try (AutoLock lock = cubeMapLock.lockForRead()) {
             return cubeMap.get(cubeName);
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java 
b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
index 360818b..17eebd6 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
@@ -132,7 +132,7 @@ public class MetadataCleanupJob {
 
         // exclude resources in use
         Set<String> activeResources = Sets.newHashSet();
-        for (CubeInstance cube : cubeManager.listAllCubes()) {
+        for (CubeInstance cube : cubeManager.reloadAndListAllCubes()) {
             activeResources.addAll(cube.getSnapshots().values());
             for (CubeSegment segment : cube.getSegments()) {
                 activeResources.addAll(segment.getSnapshotPaths());
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
 
b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 4c8c426..c29c4de 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -81,7 +81,7 @@ public class StorageCleanJobHbaseUtil {
         }
 
         // remove every segment htable from drop list
-        for (CubeInstance cube : cubeMgr.listAllCubes()) {
+        for (CubeInstance cube : cubeMgr.reloadAndListAllCubes()) {
             for (CubeSegment seg : cube.getSegments()) {
                 String tablename = seg.getStorageLocationIdentifier();
                 if (allTablesNeedToBeDropped.contains(tablename)) {
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java 
b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index b73e916..3b2e787 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -256,7 +256,7 @@ public class StorageCleanupJob extends AbstractApplication {
         }
 
         // remove every segment working dir from deletion list
-        for (CubeInstance cube : cubeMgr.listAllCubes()) {
+        for (CubeInstance cube : cubeMgr.reloadAndListAllCubes()) {
             for (CubeSegment seg : cube.getSegments()) {
                 String jobUuid = seg.getLastBuildJobID();
                 if (jobUuid != null && jobUuid.equals("") == false) {