[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4712 Optimize Optimize CubeMetaIngester.java CLI

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this 
push:
     new cd03d6d  KYLIN-4712 Optimize Optimize CubeMetaIngester.java CLI
cd03d6d is described below

commit cd03d6d41b6cec6756de1247fc9ae74dfe122685
Author: yaqian.zhang <598593...@qq.com>
AuthorDate: Tue Sep 15 17:35:12 2020 +0800

    KYLIN-4712 Optimize Optimize CubeMetaIngester.java CLI
---
 .../java/org/apache/kylin/tool/CubeMetaIngester.java | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java 
b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
index 40971c1..dfd6a84 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
@@ -33,6 +33,7 @@ import org.apache.kylin.common.persistence.ResourceTool;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.RandomUtil;
 import org.apache.kylin.common.util.ZipFileUtils;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeInstance;
@@ -75,12 +76,15 @@ public class CubeMetaIngester extends AbstractApplication {
     @SuppressWarnings("static-access")
     private static final Option OPTION_OVERWRITE_TABLES = 
OptionBuilder.withArgName("overwriteTables").hasArg().isRequired(false).withDescription("If
 table meta conflicts, overwrite the one in metadata store with the one in 
srcPath. Use in caution because it might break existing cubes! Suggest to 
backup metadata store first").create("overwriteTables");
 
+    @SuppressWarnings("static-access")
+    private static final Option OPTION_CREATE_PROJECT = 
OptionBuilder.withArgName("createProjectIfNotExists").hasArg().isRequired(false).withDescription("If
 project is not exists, kylin will create 
it.").create("createProjectIfNotExists");
     private KylinConfig kylinConfig;
 
     Set<String> requiredResources = Sets.newLinkedHashSet();
     private String targetProjectName;
     private boolean overwriteTables = false;
     private boolean forceIngest = false;
+    private boolean createProjectIfNotExists = false;
 
     @Override
     protected Options getOptions() {
@@ -89,6 +93,7 @@ public class CubeMetaIngester extends AbstractApplication {
         options.addOption(OPTION_PROJECT);
         options.addOption(OPTION_FORCE_INGEST);
         options.addOption(OPTION_OVERWRITE_TABLES);
+        options.addOption(OPTION_CREATE_PROJECT);
         return options;
     }
 
@@ -104,6 +109,10 @@ public class CubeMetaIngester extends AbstractApplication {
             overwriteTables = 
Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_OVERWRITE_TABLES));
         }
 
+        if (optionsHelper.hasOption(OPTION_CREATE_PROJECT)) {
+            createProjectIfNotExists = 
Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_CREATE_PROJECT));
+        }
+
         targetProjectName = optionsHelper.getOptionValue(OPTION_PROJECT);
 
         String srcPath = optionsHelper.getOptionValue(OPTION_SRC);
@@ -154,7 +163,7 @@ public class CubeMetaIngester extends AbstractApplication {
 
     }
 
-    private void checkAndMark(TableMetadataManager srcMetadataManager, 
DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager 
srcCubeManager, CubeDescManager srcCubeDescManager) {
+    private void checkAndMark(TableMetadataManager srcMetadataManager, 
DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager 
srcCubeManager, CubeDescManager srcCubeDescManager) throws IOException {
         if (srcHybridManager.listHybridInstances().size() > 0) {
             throw new IllegalStateException("Does not support ingest hybrid 
yet");
         }
@@ -162,7 +171,11 @@ public class CubeMetaIngester extends AbstractApplication {
         ProjectManager projectManager = 
ProjectManager.getInstance(kylinConfig);
         ProjectInstance targetProject = 
projectManager.getProject(targetProjectName);
         if (targetProject == null) {
-            throw new IllegalStateException("Target project does not exist in 
target metadata: " + targetProjectName);
+            if (createProjectIfNotExists) {
+                projectManager.createProject(targetProjectName, null, "This is 
a project automatically added when ingest cube", null);
+            } else {
+                throw new IllegalStateException("Target project does not exist 
in target metadata: " + targetProjectName);
+            }
         }
 
         TableMetadataManager metadataManager = 
TableMetadataManager.getInstance(kylinConfig);
@@ -179,6 +192,9 @@ public class CubeMetaIngester extends AbstractApplication {
                     logger.warn("Overwriting the old table desc: {}", 
tableDesc.getIdentity());
                 }
             }
+            tableDesc.setUuid(RandomUtil.randomUUID().toString());
+            tableDesc.setLastModified(0);
+            metadataManager.saveSourceTable(tableDesc, targetProjectName);
             requiredResources.add(tableDesc.getResourcePath());
         }