KYLIN-2459 fix implementation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/429dfbdc Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/429dfbdc Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/429dfbdc Branch: refs/heads/master Commit: 429dfbdc93371e8df2095e35298425ccab8ac7de Parents: 1b864f0 Author: Yang Li <liy...@apache.org> Authored: Sat May 20 19:52:30 2017 +0800 Committer: Roger Shi <rogershijich...@gmail.com> Committed: Sat May 20 21:35:36 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 36 ++++++---------- .../kylin/common/persistence/ResourceStore.java | 45 ++++++-------------- .../common/persistence/ResourceStoreTest.java | 11 +---- .../common/util/LocalFileMetadataTestCase.java | 7 +-- 4 files changed, 32 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index df3baf4..8930129 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -218,6 +218,16 @@ abstract public class KylinConfigBase implements Serializable { return getMetadataUrl().getIdentifier(); } + public Map<String, String> getResourceStoreImpls() { + Map<String, String> r = Maps.newLinkedHashMap(); + // ref constants in ISourceAware + r.put("", "org.apache.kylin.common.persistence.FileResourceStore"); + r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore"); + r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore"); + r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html + return r; + } + public String[] getRealizationProviders() { return getOptionalStringArray("kylin.metadata.realization-providers", // new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); @@ -456,6 +466,10 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.job.advanced-flat-table.class"); } + public String getJobTrackingURLPattern() { + return getOptional("kylin.job.tracking-url-pattern", ""); + } + // ============================================================================ // SOURCE.HIVE // ============================================================================ @@ -970,26 +984,4 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } - //ResourceStore Impl - /*public String getResourceStoreImpl() { - return getOptional("kylin.metadata.default-resource-store-impl", "org.apache.kylin.storage.hbase.HBaseResourceStore"); - }*/ - - public Map<String, String> getResourceStoreImpls() { - Map<String, String> r = Maps.newLinkedHashMap(); - // ref constants in ISourceAware - r.put("", "org.apache.kylin.common.persistence.FileResourceStore"); - r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore"); - r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore"); - r.putAll(getPropertiesByPrefix("kylin.resource.store.provider.")); - return r; - } - - public String getResourceStoreImpl() { - return getResourceStoreImpls().get(getMetadataUrl().getScheme()); - } - - public String getJobTrackingURLPattern() { - return getOptional("kylin.job.tracking-url-pattern", ""); - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 0565c66..7fb93e7 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.OptionsHelper; import org.slf4j.Logger; @@ -68,46 +69,24 @@ abstract public class ResourceStore { public static final String CUBE_STATISTICS_ROOT = "/cube_statistics"; public static final String BAD_QUERY_RESOURCE_ROOT = "/bad_query"; - protected static final String DEFAULT_STORE_NAME = "kylin_metadata"; - public static final String METASTORE_UUID_TAG = "/UUID"; private static final ConcurrentMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>(); - private static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>(); - - private static ArrayList<Class<? extends ResourceStore>> getKnownImpl(KylinConfig kylinConfig) { - if (knownImpl.isEmpty()) { - knownImpl.add(FileResourceStore.class); - try { - String implName = kylinConfig.getResourceStoreImpl(); - knownImpl.add(ClassUtil.forName(implName, ResourceStore.class)); - } catch (Throwable e) { - logger.warn("Failed to load HBaseResourceStore impl class: " + e.toString()); - } - } - return knownImpl; - } - private static ResourceStore createResourceStore(KylinConfig kylinConfig) { - List<Throwable> es = new ArrayList<Throwable>(); - logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store"); - for (Class<? extends ResourceStore> cls : getKnownImpl(kylinConfig)) { - try { - ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig); - if (!store.exists(METASTORE_UUID_TAG)) { - store.putResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0, StringEntity.serializer); - } - return store; - } catch (Throwable e) { - - es.add(e); + StorageURL metadataUrl = kylinConfig.getMetadataUrl(); + logger.info("Using metadata url " + metadataUrl + " for resource store"); + String clsName = kylinConfig.getResourceStoreImpls().get(metadataUrl.getScheme()); + try { + Class<? extends ResourceStore> cls = ClassUtil.forName(clsName, ResourceStore.class); + ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig); + if (!store.exists(METASTORE_UUID_TAG)) { + store.putResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0, StringEntity.serializer); } + return store; + } catch (Throwable e) { + throw new IllegalArgumentException("Failed to find metadata store by url: " + metadataUrl, e); } - for (Throwable exceptionOrError : es) { - logger.error("Create new store instance failed ", exceptionOrError); - } - throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl()); } public static ResourceStore getStore(KylinConfig kylinConfig) { http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java index 8aff1a3..91a9dfd 100644 --- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.NavigableSet; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.ClassUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,13 +44,13 @@ public class ResourceStoreTest { public static void testAStore(String url, KylinConfig kylinConfig) throws Exception { String oldUrl = replaceMetadataUrl(kylinConfig, url); - testAStore(getStoreByName(kylinConfig.getResourceStoreImpl(), kylinConfig)); + testAStore(ResourceStore.getStore(kylinConfig)); replaceMetadataUrl(kylinConfig, oldUrl); } public static void testPerformance(String url, KylinConfig kylinConfig) throws Exception { String oldUrl = replaceMetadataUrl(kylinConfig, url); - testPerformance(getStoreByName(kylinConfig.getResourceStoreImpl(), kylinConfig)); + testPerformance(ResourceStore.getStore(kylinConfig)); replaceMetadataUrl(kylinConfig, oldUrl); } @@ -205,10 +204,4 @@ public class ResourceStoreTest { return oldUrl; } - private static ResourceStore getStoreByName(String storeName, KylinConfig kylinConfig) throws Exception { - Class<? extends ResourceStore> cls = ClassUtil.forName(storeName, ResourceStore.class); - ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig); - return store; - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java index a44a55d..67f39b4 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java @@ -63,11 +63,12 @@ public class LocalFileMetadataTestCase extends AbstractKylinTestCase { } public static void cleanAfterClass() { - String tempTestMetadataUrl = LOCALMETA_TEMP_DATA; + File directory = new File(LOCALMETA_TEMP_DATA); try { - FileUtils.deleteDirectory(new File(tempTestMetadataUrl)); + FileUtils.deleteDirectory(directory); } catch (IOException e) { - throw new IllegalStateException("Can't delete directory " + tempTestMetadataUrl, e); + if (directory.exists() && directory.list().length > 0) + throw new IllegalStateException("Can't delete directory " + directory, e); } staticCleanupTestMetadata(); }
