[kylin] 01/02: KYLIN-4903 cache parent datasource to accelerate next layer's cuboid building

[zhangzc]

This is an automated email from the ASF dual-hosted git repository.

zhangzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 7370489ed53c044841955d9481a6fe00035aa697
Author: Zhichao Zhang <zhan...@apache.org>
AuthorDate: Wed Aug 11 12:04:00 2021 +0800

    KYLIN-4903 cache parent datasource to accelerate next layer's cuboid 
building
    
    Fix bug
---
 .../org/apache/kylin/engine/spark/job/BuildLayoutWithUpdate.java   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git 
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/BuildLayoutWithUpdate.java
 
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/BuildLayoutWithUpdate.java
index 91024b9..5309489 100644
--- 
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/BuildLayoutWithUpdate.java
+++ 
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/BuildLayoutWithUpdate.java
@@ -75,8 +75,9 @@ public class BuildLayoutWithUpdate {
 
     public void submit(JobEntity job, KylinConfig config) {
 
-        //if job's BuildSourceInfo is empty, it means this is a merge job, no 
parent dataset to persist
-        if (persistParentDataset && job.getBuildSourceInfo() != null && 
job.getBuildSourceInfo().getToBuildCuboids().size() > 1) {
+        // if job's BuildSourceInfo is null, it means this is a merge job or 
optimize job,
+        // no parent dataset to be persisted
+        if (persistParentDataset && job.getBuildSourceInfo() != null) {
             //when reuse parent dataset is enabled, ensure parent dataset is 
registered
             
if(!layout2DataSet.containsKey(job.getBuildSourceInfo().getLayoutId())){
                 logger.error("persist parent dataset is enabled, but parent 
dataset not registered");
@@ -102,7 +103,7 @@ public class BuildLayoutWithUpdate {
                     throwable = t;
                 } finally {
                     //unpersist parent dataset
-                    if (persistParentDataset && job.getBuildSourceInfo() != 
null && job.getBuildSourceInfo().getToBuildCuboids().size() > 1) {
+                    if (persistParentDataset && job.getBuildSourceInfo() != 
null) {
                         long remain = 
toBuildCuboidSize.get(job.getBuildSourceInfo().getLayoutId()).decrementAndGet();
                         if (remain == 0) {
                             
toBuildCuboidSize.remove(job.getBuildSourceInfo().getLayoutId());