KYLIN-2059 remove the time threshold on storage cleanup Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95ebc851 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95ebc851 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95ebc851
Branch: refs/heads/orderedbytes Commit: 95ebc851274953023ba092a249c1ddbbee5cd845 Parents: 334c2e0 Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Sep 30 09:13:03 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Thu Oct 6 14:44:05 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 ---- .../kylin/engine/mr/steps/MetadataCleanupJob.java | 2 +- .../kylin/storage/hbase/util/StorageCleanupJob.java | 15 ++++----------- 3 files changed, 5 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 4d1639b..99c3c5a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -780,10 +780,6 @@ abstract public class KylinConfigBase implements Serializable { return Long.parseLong(getOptional("kylin.query.hbase.hconnection.threads.alive.seconds", "60")); } - public long getStorageCleanupTimeThreshold() { - return Long.valueOf(this.getOptional("kylin.storage.cleanup.time.threshold", "172800000")); //default two days - } - public int getAppendDictEntrySize() { return Integer.parseInt(getOptional("kylin.dict.append.entry.size", "10000000")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java index 962697e..f2b1d6b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java @@ -54,7 +54,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob { private KylinConfig config = null; - public static final long TIME_THREADSHOLD = 2 * 24 * 3600 * 1000L; // 2 days + public static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour public static final long TIME_THREADSHOLD_FOR_JOB = 30 * 24 * 3600 * 1000L; // 30 days /* http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java index dffce36..2c2f11c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java @@ -34,7 +34,6 @@ import java.util.regex.Pattern; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -63,10 +62,10 @@ public class StorageCleanupJob extends AbstractApplication { @SuppressWarnings("static-access") protected static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused storage").create("delete"); - protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning: will delete any intermediate hive tables").create("force"); + protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning: will delete all kylin intermediate hive tables").create("force"); protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class); - public static final int TIME_THRESHOLD_DELETE_HTABLE = 10; // Unit minute + public static final int deleteTimeout = 10; // Unit minute protected boolean delete = false; protected boolean force = false; @@ -74,7 +73,6 @@ public class StorageCleanupJob extends AbstractApplication { private void cleanUnusedHBaseTables(Configuration conf) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); - long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold(); // get all kylin hbase tables HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; @@ -82,14 +80,9 @@ public class StorageCleanupJob extends AbstractApplication { List<String> allTablesNeedToBeDropped = new ArrayList<String>(); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); - String creationTime = desc.getValue(IRealizationConstants.HTableCreationTime); if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) { //only take care htables that belongs to self, and created more than 2 days - if (StringUtils.isEmpty(creationTime) || (System.currentTimeMillis() - Long.valueOf(creationTime) > TIME_THREADSHOLD)) { allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString()); - } else { - logger.info("Exclude table " + desc.getTableName().getNameAsString() + " from drop list, as it is newly created"); - } } } @@ -111,9 +104,9 @@ public class StorageCleanupJob extends AbstractApplication { FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin, htableName)); executorService.execute(futureTask); try { - futureTask.get(TIME_THRESHOLD_DELETE_HTABLE, TimeUnit.MINUTES); + futureTask.get(deleteTimeout, TimeUnit.MINUTES); } catch (TimeoutException e) { - logger.warn("It fails to delete htable " + htableName + ", for it cost more than " + TIME_THRESHOLD_DELETE_HTABLE + " minutes!"); + logger.warn("It fails to delete htable " + htableName + ", for it cost more than " + deleteTimeout + " minutes!"); futureTask.cancel(true); } catch (Exception e) { e.printStackTrace();
