(kylin) branch kylin5 updated: KYLIN-6018 optimize metadata migrate tool 4x-->5x, run without kylin_config

[jlfsdtc]

This is an automated email from the ASF dual-hosted git repository.

jlfsdtc pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/kylin5 by this push:
     new 0bdc1886e8 KYLIN-6018 optimize metadata migrate tool 4x-->5x, run 
without kylin_config
0bdc1886e8 is described below

commit 0bdc1886e84671d1ed07f9b0f77f4deedb14534f
Author: Zhong.Zhu <82999245+thy950...@users.noreply.github.com>
AuthorDate: Thu Nov 7 19:14:32 2024 +0800

    KYLIN-6018 optimize metadata migrate tool 4x-->5x, run without kylin_config
---
 .../metadata/MigrateKEMetadataTool.java            | 26 ++++++++++++----------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git 
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MigrateKEMetadataTool.java
 
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MigrateKEMetadataTool.java
index 520863c5b8..81b087e4a9 100644
--- 
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MigrateKEMetadataTool.java
+++ 
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MigrateKEMetadataTool.java
@@ -473,7 +473,6 @@ public class MigrateKEMetadataTool {
         }
         final Properties props = config.exportToProperties();
         KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
-        val dstResourceStore = (InMemResourceStore) 
ResourceStore.getKylinMetaStore(dstConfig);
 
         String type = inputPath.endsWith(".zip") ? 
FileSystemMetadataStore.Type.ZIP.name() : DIR.name();
         if (outputPath == null) {
@@ -487,34 +486,37 @@ public class MigrateKEMetadataTool {
             log.info("outputPath: " + outputPath);
         }
 
+        String outputZipFile = null;
+        if (outputPath.endsWith(".zip")) {
+            // metnauadata_url doesn't support zip file
+            outputZipFile = outputPath;
+            outputPath = outputPath.substring(0, outputPath.lastIndexOf("/"));
+        }
 
         Path metadataPath = new Path(outputPath);
         FileSystem fs = HadoopUtil.getWorkingFileSystem(metadataPath);
+        if (!fs.exists(metadataPath)) {
+            fs.mkdirs(metadataPath);
+        }
         if (type.equals(FileSystemMetadataStore.Type.ZIP.name())) {
-            if (!fs.exists(metadataPath.getParent())) {
-                fs.mkdirs(metadataPath.getParent());
-            }
             dstConfig.setMetadataUrl(outputPath + "@file,zip=1");
         } else {
-            if (!fs.exists(metadataPath)) {
-                fs.mkdirs(metadataPath);
-            }
             dstConfig.setMetadataUrl(outputPath);
         }
+        val dstResourceStore = (InMemResourceStore) 
ResourceStore.getKylinMetaStore(dstConfig);
 
         FileSystemMetadataStore outputMetadataStore = 
(FileSystemMetadataStore) MetadataStore
                 .createMetadataStore(dstConfig);
         MetadataStore.MemoryMetaData memoryMetaData = 
loadOldMetaData(outputMetadataStore, inputPath, type);
         dstResourceStore.resetData(memoryMetaData);
-        dumpToNewFormat(outputMetadataStore, dstResourceStore, type);
+        dumpToNewFormat(outputMetadataStore, dstResourceStore, type, 
outputZipFile);
     }
 
-    private void dumpToNewFormat(FileSystemMetadataStore fileStore, 
ResourceStore resourceStore, String type)
-            throws IOException, InterruptedException, ExecutionException {
+    private void dumpToNewFormat(FileSystemMetadataStore fileStore, 
ResourceStore resourceStore, String type,
+            String outputZipFile) throws IOException, InterruptedException, 
ExecutionException {
         val resources = 
resourceStore.listResourcesRecursively(MetadataType.ALL.name());
         if (FileSystemMetadataStore.Type.ZIP.name().equals(type)) {
-            fileStore.dumpToZip(resourceStore, resources,
-                    new 
Path(resourceStore.getConfig().getMetadataUrl().getIdentifier()));
+            fileStore.dumpToZip(resourceStore, resources, new 
Path(outputZipFile));
         } else {
             fileStore.dumpToFile(resourceStore, resources);
         }