Repository: kylin Updated Branches: refs/heads/master 4b00ec26d -> eb18ac59f
KYLIN-2304 Only copy latest version dict for global dict Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb18ac59 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb18ac59 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb18ac59 Branch: refs/heads/master Commit: eb18ac59fa74b3f2f1f0163f7728739a1027a233 Parents: 4b00ec2 Author: kangkaisen <kangkai...@live.com> Authored: Sun Dec 18 20:09:03 2016 +0800 Committer: kangkaisen <kangkai...@163.com> Committed: Sat Jan 14 19:07:59 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/dict/AppendTrieDictionary.java | 19 +++++++++++++------ .../org/apache/kylin/dict/CachedTreeMap.java | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 5029bc4..80403cc 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -1173,13 +1173,15 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { @Override public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException { + //copy appendDict Configuration conf = new Configuration(); - AppendTrieDictionary newDict = new AppendTrieDictionary(); - newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConvert); - newDict.initDictSliceMap((CachedTreeMap)dictSliceMap); - logger.info("Copy AppendDict from {} to {}", this.baseDir, newDict.baseDir); - Path srcPath = new Path(this.baseDir); - Path dstPath = new Path(newDict.baseDir); + + Path base = new Path(baseDir); + FileSystem srcFs = FileSystem.get(base.toUri(), conf); + Path srcPath = CachedTreeMap.getLatestVersion(conf, srcFs, base); + Path dstPath = new Path(srcPath.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory())); + logger.info("Copy appendDict from {} to {}", srcPath, dstPath); + FileSystem dstFs = FileSystem.get(dstPath.toUri(), conf); if (dstFs.exists(dstPath)) { logger.info("Delete existing AppendDict {}", dstPath); @@ -1187,6 +1189,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { } FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf); + // init new AppendTrieDictionary + AppendTrieDictionary newDict = new AppendTrieDictionary(); + newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConverter); + newDict.initDictSliceMap((CachedTreeMap) dictSliceMap); + return newDict; } http://git-wip-us.apache.org/repos/asf/kylin/blob/eb18ac59/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java index 3be4fd7..cc23261 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java @@ -225,7 +225,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext return getLatestVersion(conf, fs, baseDir).toUri().getPath(); } - private static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException { + public static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException { String[] versions = listAllVersions(fs, baseDir); if (versions.length > 0) { return new Path(versions[versions.length - 1]);
