This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9b8050e00af23094104bf35439a5fd36470aa23a
Author: Hang Jia <754332...@qq.com>
AuthorDate: Thu Aug 31 11:13:06 2023 +0800
KYLIN-5802 Models with the same name exist when restore metadata
---
.../kylin/common/exception/code/ErrorCodeTool.java | 3 +-
.../resources/kylin_error_msg_conf_cn.properties | 1 +
.../resources/kylin_error_msg_conf_en.properties | 1 +
.../main/resources/kylin_errorcode_conf.properties | 1 +
.../apache/kylin/helper/MetadataToolHelper.java | 118 +++++++++++++++++++++
.../org/apache/kylin/tool/MetadataToolTest.java | 57 ++++++++++
6 files changed, 180 insertions(+), 1 deletion(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
index 61cf7d13cd..6a7a46bee4 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
@@ -24,7 +24,8 @@ public enum ErrorCodeTool implements ErrorCodeProducer {
"KE-050040203"), PARAMETER_TIMESTAMP_COMPARE("KE-050040204"),
// path & file
- PATH_NOT_EXISTS("KE-050041201"), FILE_ALREADY_EXISTS("KE-050041202");
+ PATH_NOT_EXISTS("KE-050041201"), FILE_ALREADY_EXISTS("KE-050041202"),
+ MODEL_DUPLICATE_UUID_FAILED("KE-050041203");
private final ErrorCode errorCode;
private final ErrorMsg errorMsg;
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 010c2b10c3..2e19879659 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -226,6 +226,7 @@ KE-050040204=参数 “-endTime” <= 参数 “-startTime” 。
## 500412XX
KE-050041201=该路径不存在：%s。
KE-050041202=该路径已存在：%s。
+KE-050041203=请修改模型名称后再恢复:[project]:models: %s
# Common
## KE-060100201
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 75f05b384b..f9d66504ca 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -226,6 +226,7 @@ KE-050040204=Parameter "-endTime" <= Parameter "-startTime".
## 500412XX
KE-050041201=The path does not exist: %s.
KE-050041202=The path already exists: %s.
+KE-050041203=Please modify the model name and then restore:[project]:models: %s
# Common
## KE-060100201
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index f8ed420188..f8dd010445 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -237,6 +237,7 @@ KE-050040204
## 500412XX
KE-050041201
KE-050041202
+KE-050041203
# Common
KE-060100201
diff --git
a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index f000e766a6..f3d920d918 100644
--- a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
+++ b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -19,6 +19,7 @@
package org.apache.kylin.helper;
import static
org.apache.kylin.common.exception.code.ErrorCodeTool.FILE_ALREADY_EXISTS;
+import static
org.apache.kylin.common.exception.code.ErrorCodeTool.MODEL_DUPLICATE_UUID_FAILED;
import java.io.File;
import java.io.IOException;
@@ -28,9 +29,12 @@ import java.nio.file.Paths;
import java.time.Clock;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.NavigableSet;
import java.util.Objects;
import java.util.Set;
@@ -58,8 +62,10 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.tool.HDFSMetadataTool;
import org.apache.kylin.tool.constant.StringConstant;
@@ -288,6 +294,7 @@ public class MetadataToolHelper {
public void restore(ResourceStore currentResourceStore, ResourceStore
restoreResourceStore, String project,
boolean delete) {
+ checkDuplicateUuidModel(currentResourceStore, restoreResourceStore,
project, delete);
if (StringUtils.isBlank(project)) {
logger.info("start to restore all projects");
var srcProjectFolders = restoreResourceStore.listResources("/");
@@ -340,6 +347,117 @@ public class MetadataToolHelper {
logger.info("restore successfully");
}
+ public void checkDuplicateUuidModel(ResourceStore currentResourceStore,
ResourceStore restoreResourceStore,
+ String project, boolean delete) {
+ logger.info("start check duplicate uuid model");
+ if (delete) {
+ return;
+ }
+
+ Map<String, List<String>> duplicateUuidModelByProject =
getDuplicateUuidModelByProject(currentResourceStore,
+ restoreResourceStore, project);
+
+ if (!duplicateUuidModelByProject.isEmpty()) {
+ String errorMsg = duplicateUuidModelByProject.entrySet().stream()
+ .map(m -> "[" + m.getKey() + "]:" + String.join(",",
m.getValue()))
+ .collect(Collectors.joining(";"));
+ String info = String.format(
+ "[UNEXPECTED_THINGS_HAPPENED] There will be models with
the same name after recovery, please rename these models
first:[project]:models: %s ",
+ errorMsg);
+ logger.error(info);
+ System.out.println(StringConstant.ANSI_RED + info +
StringConstant.ANSI_RESET);
+ throw new KylinException(MODEL_DUPLICATE_UUID_FAILED, errorMsg);
+ }
+ logger.info("end check duplicate uuid model");
+ }
+
+ private Map<String, List<String>>
getDuplicateUuidModelByProject(ResourceStore currentResourceStore,
+ ResourceStore restoreResourceStore, String project) {
+ Map<String, List<String>> duplicateUuidModelByProject =
Maps.newHashMap();
+ if (StringUtils.isBlank(project)) {
+ duplicateUuidModelByProject =
getDuplicateUuidModelByAllProject(currentResourceStore, restoreResourceStore);
+ } else {
+ List<String> duplicateUuidModel =
getDuplicateUuidModel(currentResourceStore, restoreResourceStore,
+ project);
+ if (!duplicateUuidModel.isEmpty()) {
+ duplicateUuidModelByProject.put(project, duplicateUuidModel);
+ }
+ }
+ return duplicateUuidModelByProject;
+ }
+
+ private Map<String, List<String>>
getDuplicateUuidModelByAllProject(ResourceStore currentResourceStore,
+ ResourceStore restoreResourceStore) {
+ var destProjectFolders = currentResourceStore.listResources("/");
+ var srcProjectFolders = restoreResourceStore.listResources("/");
+ destProjectFolders = destProjectFolders == null ? Sets.newTreeSet() :
destProjectFolders;
+ srcProjectFolders = srcProjectFolders == null ? Sets.newTreeSet() :
srcProjectFolders;
+ val projectFolders = Sets.union(srcProjectFolders, destProjectFolders);
+
+ Map<String, List<String>> duplicateUuidModelByProject =
Maps.newHashMap();
+ for (String projectPath : projectFolders) {
+ if (projectPath.equals(ResourceStore.METASTORE_UUID_TAG)
+ || projectPath.equals(ResourceStore.METASTORE_IMAGE)) {
+ continue;
+ }
+ val projectName = Paths.get(projectPath).getName(0).toString();
+ List<String> duplicateUuidModel =
getDuplicateUuidModel(currentResourceStore, restoreResourceStore,
+ projectName);
+ if (!duplicateUuidModel.isEmpty()) {
+ duplicateUuidModelByProject.put(projectName,
duplicateUuidModel);
+ }
+ }
+ return duplicateUuidModelByProject;
+ }
+
+ private List<String> getDuplicateUuidModel(ResourceStore
currentResourceStore, ResourceStore restoreResourceStore,
+ String projectName) {
+ String modelDescRootPath = File.separator + projectName +
ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT;
+ Set<String> destModelResource =
currentResourceStore.listResources(modelDescRootPath);
+ Set<String> srcModelResource =
restoreResourceStore.listResources(modelDescRootPath);
+ destModelResource = destModelResource == null ? Collections.emptySet()
: destModelResource;
+ srcModelResource = srcModelResource == null ? Collections.emptySet() :
srcModelResource;
+
+ Sets.SetView<String> insertsModelResource =
Sets.difference(srcModelResource, destModelResource);
+
+ List<NDataModel> allModels = new ArrayList<>(
+ getModelListFromResource(projectName, destModelResource,
currentResourceStore));
+ List<NDataModel> insertsModels = getModelListFromResource(projectName,
new HashSet<>(insertsModelResource),
+ restoreResourceStore);
+ allModels.addAll(insertsModels);
+
+ Map<String, Set<String>> nameUuids = Maps.newHashMap();
+ for (NDataModel model : allModels) {
+ String modelAlias = model.getAlias();
+ nameUuids.putIfAbsent(modelAlias, Sets.newHashSet());
+ nameUuids.get(modelAlias).add(model.getUuid());
+ }
+ return nameUuids.entrySet().stream().filter(m -> m.getValue().size() >
1).map(Map.Entry::getKey)
+ .collect(Collectors.toList());
+ }
+
+ public List<NDataModel> getModelListFromResource(String projectName,
Set<String> modelResource,
+ ResourceStore resourceStore) {
+
+ if (modelResource == null) {
+ return new ArrayList<>();
+ }
+ List<NDataModel> models = new ArrayList<>();
+ for (String resource : modelResource) {
+ try {
+ NDataModel nDataModel =
JsonUtil.readValue(resourceStore.getResource(resource).getByteSource().read(),
+ NDataModel.class);
+ nDataModel.setProject(projectName);
+ models.add(nDataModel);
+ } catch (IOException e) {
+ if (!KylinConfig.getInstanceFromEnv().isUTEnv()) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+ return models;
+ }
+
private int doRestore(ResourceStore currentResourceStore, ResourceStore
restoreResourceStore,
Set<String> destResources, Set<String> srcResources, boolean
delete) throws IOException {
val threadViewRS =
ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
index b90139a94e..3bb9652edb 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.tool;
import static
org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil.datasourceParameters;
+import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -64,6 +65,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.OptionBuilder;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
@@ -452,6 +454,61 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
}
+ @Test
+ public void testRestoreDuplicateUuidModel() throws Exception {
+ val restoreFolder = temporaryFolder.newFolder();
+ MetadataToolTestFixture.fixtureRestoreTest(getTestConfig(),
restoreFolder, "/");
+
+ val tool = new MetadataTool(getTestConfig());
+ tool.execute(new String[] { "-restore", "-dir",
restoreFolder.getAbsolutePath() });
+ NDataModelManager dataModelManager =
NDataModelManager.getInstance(getTestConfig(), "default");
+
+ String modelId = "82fa7671-a935-45f5-8779-85703601f49a";
+ NDataModel dataModelDesc = dataModelManager.getDataModelDesc(modelId);
+ dataModelManager.dropModel(dataModelDesc.getUuid());
+ NDataModel nDataModel =
dataModelManager.copyBySerialization(dataModelDesc);
+ nDataModel.setMvcc(-1);
+ nDataModel.setUuid(RandomUtil.randomUUIDStr());
+ dataModelManager.createDataModelDesc(nDataModel,
nDataModel.getOwner());
+
+ String modelId2 = "a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94";
+ NDataModel dataModelDesc2 =
dataModelManager.getDataModelDesc(modelId2);
+ dataModelManager.dropModel(dataModelDesc2.getUuid());
+ NDataModel nDataModel2 =
dataModelManager.copyBySerialization(dataModelDesc2);
+ nDataModel2.setMvcc(-1);
+ nDataModel2.setUuid(RandomUtil.randomUUIDStr());
+ dataModelManager.createDataModelDesc(nDataModel2,
nDataModel2.getOwner());
+
+ try {
+ tool.execute(new String[] { "-restore", "-dir",
restoreFolder.getAbsolutePath() });
+ } catch (Exception e) {
+ assertTrue(e instanceof KylinException);
+ assertEquals(
+ "KE-050041203: Please modify the model name and then
restore:[project]:models: [default]:ut_inner_join_cube_partial,test_encoding",
+ e.getCause().toString());
+ }
+ long start = System.currentTimeMillis() / 1000;
+ await().until(() -> start != (System.currentTimeMillis() / 1000));
+ try {
+ tool.execute(new String[] { "-restore", "-project", "default",
"-dir", restoreFolder.getAbsolutePath() });
+ } catch (Exception e) {
+ assertTrue(e instanceof KylinException);
+ assertEquals(
+ "KE-050041203: Please modify the model name and then
restore:[project]:models: [default]:ut_inner_join_cube_partial,test_encoding",
+ e.getCause().toString());
+ }
+
+
Assertions.assertThat(dataModelManager.getDataModelDesc(modelId)).isNull();
+
Assertions.assertThat(dataModelManager.getDataModelDesc(modelId2)).isNull();
+
+ long start2 = System.currentTimeMillis() / 1000;
+ await().until(() -> start2 != (System.currentTimeMillis() / 1000));
+ tool.execute(new String[] { "-restore", "-dir",
restoreFolder.getAbsolutePath(), "--after-truncate" });
+
+
Assertions.assertThat(dataModelManager.getDataModelDesc(modelId)).isNotNull();
+
Assertions.assertThat(dataModelManager.getDataModelDesc(modelId2)).isNotNull();
+ }
+
@Test
public void testRestoreOverwriteAllCompressWithSrcOrDestIsEmpty() throws
Exception {
val emptyFolder = temporaryFolder.newFolder();
