[kylin] branch master updated: KYLIN-4686 clean metadata support to delete all jobs

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new 2e937c6  KYLIN-4686 clean metadata support to delete all jobs
2e937c6 is described below

commit 2e937c62990ecbc9f77f6f377b7bd4cf1dbdc230
Author: chuxiao <chux...@didichuxing.com>
AuthorDate: Thu Aug 6 11:10:20 2020 +0800

    KYLIN-4686 clean metadata support to delete all jobs
---
 .../main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java  | 4 ++--
 .../java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java   | 4 ++--
 tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java | 9 ++++++++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java 
b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
index 78302bb..a9a7f12 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
@@ -80,7 +80,7 @@ public class MetadataCleanupJob {
     }
 
     // function entrance
-    public Map<String, Long> cleanup(boolean delete, int jobOutdatedDays) 
throws Exception {
+    public Map<String, Long> cleanup(boolean delete, int jobOutdatedDays, 
boolean deleteAllJobs) throws Exception {
         CubeManager cubeManager = CubeManager.getInstance(config);
         long newResourceTimeCut = System.currentTimeMillis() - 
NEW_RESOURCE_THREADSHOLD_MS;
         FileSystem fs = 
HadoopUtil.getWorkingFileSystem(HadoopUtil.getCurrentConfiguration());
@@ -168,7 +168,7 @@ public class MetadataCleanupJob {
         List<ExecutablePO> allExecutable = executableDao.getJobs();
         for (ExecutablePO executable : allExecutable) {
             long lastModified = executable.getLastModified();
-            if (lastModified < outdatedJobTimeCut && 
isJobComplete(executableDao, executable)) {
+            if (lastModified < outdatedJobTimeCut && (deleteAllJobs || 
isJobComplete(executableDao, executable))) {
                 String jobResPath = ResourceStore.EXECUTE_RESOURCE_ROOT + "/" 
+ executable.getUuid();
                 String jobOutputResPath = 
ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid();
                 long outputLastModified = getTimestamp(jobOutputResPath);
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
 
b/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
index ad819e6..bed1a9c 100644
--- 
a/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
+++ 
b/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
@@ -45,7 +45,7 @@ public class MetadataCleanupJobTest {
         // file resource store may lose timestamp precision with millis 
second, set last modified as 2000
         staticCreateTestMetadata(false, new ResetTimeHook(2000, 
"src/test/resources/test_meta"));
         MetadataCleanupJob metadataCleanupJob = new MetadataCleanupJob();
-        Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30);
+        Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30, 
false);
         Assert.assertEquals(7, cleanupMap.size());
         for (long timestamp : cleanupMap.values()) {
             Assert.assertEquals(2000, timestamp);
@@ -56,7 +56,7 @@ public class MetadataCleanupJobTest {
     public void testNotCleanUp() throws Exception {
         staticCreateTestMetadata(false, new 
ResetTimeHook(System.currentTimeMillis(), "src/test/resources/test_meta"));
         MetadataCleanupJob metadataCleanupJob = new MetadataCleanupJob();
-        Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30);
+        Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30, 
false);
         Assert.assertEquals(0, cleanupMap.size());
     }
 
diff --git a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java 
b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
index 1e5214d..1cb5df7 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
@@ -39,6 +39,11 @@ public class MetadataCleanupJob extends AbstractApplication {
             .isRequired(false).withDescription("Specify how many days of job 
metadata keeping. Default 30 days")
             .create("jobThreshold");
 
+    @SuppressWarnings("static-access")
+    private static final Option OPTION_DELETE_ALL_JOBS = 
OptionBuilder.withArgName("deleteAllJobs").hasArg()
+            .isRequired(false).withDescription("Delte all jobs include error 
jobs. Default false")
+            .create("deleteAllJobs");
+
     public static void main(String[] args) throws Exception {
         new MetadataCleanupJob().execute(args);
     }
@@ -48,6 +53,7 @@ public class MetadataCleanupJob extends AbstractApplication {
         Options options = new Options();
         options.addOption(OPTION_DELETE);
         options.addOption(OPTION_THRESHOLD_FOR_JOB);
+        options.addOption(OPTION_DELETE_ALL_JOBS);
         return options;
     }
 
@@ -59,7 +65,8 @@ public class MetadataCleanupJob extends AbstractApplication {
         int jobOutdatedDays = optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB)
                 ? 
Integer.parseInt(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB))
                 : DEFAULT_JOB_OUTDATED_THRESHOLD_DAYS;
+        boolean deleteAllJobs = 
Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE_ALL_JOBS));
         
-        new org.apache.kylin.rest.job.MetadataCleanupJob().cleanup(delete, 
jobOutdatedDays);
+        new org.apache.kylin.rest.job.MetadataCleanupJob().cleanup(delete, 
jobOutdatedDays, deleteAllJobs);
     }
 }