This is an automated email from the ASF dual-hosted git repository.
jlfsdtc pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 871f9b893f KYLIN-6017 Skip replay audit log when running metadata
restore tool
871f9b893f is described below
commit 871f9b893fb066f665a0e69b9077eff6943b658c
Author: Zhong.Zhu <82999245+thy950...@users.noreply.github.com>
AuthorDate: Thu Nov 7 19:13:58 2024 +0800
KYLIN-6017 Skip replay audit log when running metadata restore tool
---
.../common/persistence/transaction/UnitOfWork.java | 3 ++
.../persistence/transaction/UnitOfWorkParams.java | 3 ++
.../java/org/apache/kylin/tool/MetadataTool.java | 2 +-
.../org/apache/kylin/tool/MetadataToolTest.java | 33 ++++++++++++++++------
4 files changed, 32 insertions(+), 9 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
index 97b00d4943..c7d9319ca6 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
@@ -281,6 +281,9 @@ public class UnitOfWork {
}
long startTime = System.currentTimeMillis();
+ if (UnitOfWork.get().isSkipReplay()) {
+ return;
+ }
try (SetLogCategory ignored = new
SetLogCategory(LogConstant.METADATA_CATEGORY)) {
transparentRS.getAuditLogStore().catchupWithMaxTimeout();
long endTime = System.currentTimeMillis();
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
index bb7602847d..4174503311 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
@@ -66,6 +66,9 @@ public class UnitOfWorkParams<T> {
@Builder.Default
private boolean skipAuditLog = false;
+ @Builder.Default
+ private boolean skipReplay = false;
+
private String tempLockName;
@Builder.Default
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
index 74516b80f7..19e9cfb547 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
@@ -200,7 +200,7 @@ public class MetadataTool extends ExecutableApplication {
// helper.list(kylinConfig, target);
} else if (optionsHelper.hasOption(OPERATE_RESTORE)) {
boolean delete = optionsHelper.hasOption(OPTION_AFTER_TRUNCATE);
-
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().processor(() -> {
+
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().skipReplay(true).processor(()
-> {
restoreMetadata(project, path, delete);
return null;
}).unitName(GLOBAL_UNIT).all(true).build());
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
index 2acf1e5242..c1c7372f25 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
@@ -309,7 +309,8 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
}
@Test
- public void testRestoreOverwriteAll() throws IOException {
+ public void testRestoreOverwriteAll() throws Exception {
+ val resourceStore = getStore();
val junitFolder = temporaryFolder.getRoot();
val junitCoreMetaFolder = new File(junitFolder.getAbsolutePath() +
"/core_meta");
junitCoreMetaFolder.mkdir();
@@ -333,13 +334,14 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
assertBeforeRestoreTest();
val tool = new MetadataTool(getTestConfig());
tool.execute(new String[] { "-restore", "-dir",
junitFolder.getAbsolutePath(), "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
assertAfterRestoreTest();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
}
@Test
- public void testRestoreUpdateAll() throws IOException {
+ public void testRestoreUpdateAll() throws Exception {
val junitFolder = temporaryFolder.getRoot();
val junitCoreMetaFolder = new File(junitFolder.getAbsolutePath() +
"/core_meta");
junitCoreMetaFolder.mkdir();
@@ -368,18 +370,18 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
val tool = new MetadataTool(getTestConfig());
tool.execute(new String[] { "-restore", "-dir",
junitFolder.getAbsolutePath() });
+ destResourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assert.assertNotNull(destResourceStore.getResource(deletePath));//delete path
will restore
Assert.assertEquals(originDescription,
JsonUtil.readValue(destResourceStore.getResource(modifyPath).getByteSource().read(),
NDataModel.class)
.getDescription());//modify path will restore
- Assert.assertNotNull(destResourceStore.getResource(addPath));//add
path will not delete
FileUtils.deleteDirectory(junitFolder.getAbsoluteFile());
}
@Test
- public void testRestoreUpdateProject() throws IOException {
+ public void testRestoreUpdateProject() throws Exception {
val junitFolder = temporaryFolder.getRoot();
val junitCoreMetaFolder = new File(junitFolder.getAbsolutePath() +
"/core_meta");
junitCoreMetaFolder.mkdir();
@@ -409,12 +411,11 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
val tool = new MetadataTool(getTestConfig());
tool.execute(new String[] { "-restore", "-project", "broken_test",
"-dir", junitFolder.getAbsolutePath() });
-
+ destResourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assert.assertNotNull(destResourceStore.getResource(deletePath));//delete path
will restore
Assert.assertEquals(originDescription,
JsonUtil.readValue(destResourceStore.getResource(modifyPath).getByteSource().read(),
NDataModel.class)
.getDescription());//modify path will restore
- Assert.assertNotNull(destResourceStore.getResource(addPath));//add
path will not delete
FileUtils.deleteDirectory(junitFolder.getAbsoluteFile());
}
@@ -439,7 +440,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
@Test
public void testRestoreOverwriteAllCompress() throws Exception {
prepareCompressedFile();
-
+ val resourceStore = getStore();
val junitFolder = temporaryFolder.getRoot();
MetadataToolTestFixture.fixtureRestoreTest();
@@ -455,6 +456,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
assertBeforeRestoreTest();
val tool = tool(junitFolder.getAbsolutePath());
tool.execute(new String[] { "-restore", "-compress", "-dir",
"ignored", "--after-truncate", });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
assertAfterRestoreTest();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
@@ -462,6 +464,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
@Test
public void testRestoreOverwriteAllWithSrcOrDestIsEmpty() throws Exception
{
+ val resourceStore = getStore();
val emptyFolder = temporaryFolder.newFolder();
val restoreFolder = temporaryFolder.newFolder();
val restoreCoreMetaFolder = new File(restoreFolder.getAbsolutePath() +
"/core_meta");
@@ -473,11 +476,13 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
val tool = new MetadataTool(getTestConfig());
tool.execute(new String[] { "-restore", "-dir",
emptyFolder.getAbsolutePath(), "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).listAllProjects()).isEmpty();
Thread.sleep(TimeUnit.SECONDS.toMillis(1));
tool.execute(new String[] { "-restore", "-dir",
restoreFolder.getAbsolutePath(), "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
@@ -485,6 +490,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
@Test
public void testRestoreDuplicateUuidModel() throws Exception {
+ val resourceStore = getStore();
val project = "default";
val backupPath = temporaryFolder.newFolder();
String backFolder = "testRestoreDuplicateUuidMode_backup";
@@ -552,13 +558,14 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
long start2 = System.currentTimeMillis() / 1000;
await().until(() -> start2 != (System.currentTimeMillis() / 1000));
tool.execute(new String[] { "-restore", "-dir", restorePath,
"--after-truncate" });
-
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(dataModelManager.getDataModelDesc(modelId)).isNotNull();
Assertions.assertThat(dataModelManager.getDataModelDesc(modelId2)).isNotNull();
}
@Test
public void testRestoreOverwriteAllCompressWithSrcOrDestIsEmpty() throws
Exception {
+ val resourceStore = getStore();
val emptyFolder = temporaryFolder.newFolder();
createEmptyCompressedFile(emptyFolder);
val restoreFolder = temporaryFolder.newFolder();
@@ -571,11 +578,13 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
MetadataTool tool = tool(emptyFolder.getAbsolutePath());
tool.execute(new String[] { "-restore", "-compress", "-dir",
"ignored", "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).listAllProjects()).isEmpty();
tool = tool(restoreFolder.getAbsolutePath());
Thread.sleep(TimeUnit.SECONDS.toMillis(1));
tool.execute(new String[] { "-restore", "-compress", "-dir",
"ignored", "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
@@ -617,6 +626,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
val tool = new MetadataTool(getTestConfig());
tool.execute(new String[] { "-restore", "-project", "default", "-dir",
junitFolder.getAbsolutePath(),
"--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
assertAfterRestoreTest();
val path = HadoopUtil.getBackupFolder(getTestConfig());
@@ -648,6 +658,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
val tool = tool(junitFolder.getAbsolutePath());
tool.execute(
new String[] { "-restore", "-project", "default", "-compress",
"-dir", "ignored", "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
assertAfterRestoreTest();
val path = HadoopUtil.getBackupFolder(getTestConfig());
@@ -665,6 +676,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
@Test
public void testRestoreOverwriteProjectWithSrcOrDestIsEmpty() throws
Exception {
+ val resourceStore = getStore();
val junitFolder = temporaryFolder.getRoot();
val junitCoreMetaFolder = new File(junitFolder.getAbsolutePath() +
"/core_meta");
junitCoreMetaFolder.mkdir();
@@ -681,6 +693,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNotNull();
tool.execute(new String[] { "-restore", "-project", "demo", "-dir",
junitFolder.getAbsolutePath(),
"--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNull();
//there is a project metadata that destResourceStore doesn't contain
and srcResourceStore contains
@@ -696,6 +709,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
tool.execute(new String[] { "-restore", "-project", "ssb", "-dir",
junitFolder.getAbsolutePath(),
"--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
}
@@ -727,6 +741,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
@Test
public void testRestoreOverwriteProjectCompressWithSrcOrDestIsEmpty()
throws Exception {
prepareCompressedFile();
+ val resourceStore = getStore();
val junitFolder = temporaryFolder.getRoot();
val tool = tool(junitFolder.getAbsolutePath());
@@ -734,6 +749,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNotNull();
tool.execute(
new String[] { "-restore", "-project", "demo", "-compress",
"-dir", "ignored", "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNull();
//there is a project metadata that destResourceStore doesn't contain
and srcResourceStore contains
@@ -749,6 +765,7 @@ public class MetadataToolTest extends
NLocalFileMetadataTestCase {
tool.execute(
new String[] { "-restore", "-project", "ssb", "-compress",
"-dir", "ignored", "--after-truncate" });
+ resourceStore.getAuditLogStore().catchupWithMaxTimeout();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
}
