KYLIN-1714 Make job/source/storage engines configurable from kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f31ad6e5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f31ad6e5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f31ad6e5 Branch: refs/heads/master Commit: f31ad6e5bc5f4f7e90de04e6ad21f0ad11b45c52 Parents: aa7558a Author: Li Yang <liy...@apache.org> Authored: Fri May 20 15:00:07 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri May 20 15:00:07 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 13 +++++++++ .../apache/kylin/common/KylinConfigBase.java | 28 ++++++++++++++------ .../org/apache/kylin/engine/EngineFactory.java | 8 +----- .../kylin/metadata/model/IEngineAware.java | 2 +- .../kylin/metadata/model/IStorageAware.java | 2 +- .../org/apache/kylin/source/SourceFactory.java | 7 ++--- .../apache/kylin/storage/StorageFactory.java | 11 ++------ .../test_case_data/localmeta/kylin.properties | 12 +++++++++ .../test_case_data/sandbox/kylin.properties | 13 +++++++++ 9 files changed, 65 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 9a2fd81..b4a49f6 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -15,6 +15,18 @@ # limitations under the License. # +# job engines +kylin.job.engine.0=org.apache.kylin.engine.mr.MRBatchCubingEngine +kylin.job.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2 + +# source engines +kylin.source.engine.0=org.apache.kylin.source.hive.HiveSource + +# storage engines +kylin.storage.engine.0=org.apache.kylin.storage.hbase.HBaseStorage +kylin.storage.engine.1=org.apache.kylin.storage.hybrid.HybridStorage +kylin.storage.engine.2=org.apache.kylin.storage.hbase.HBaseStorage + # kylin server's mode kylin.server.mode=all @@ -89,6 +101,7 @@ kylin.query.security.enabled=true # whether get job status from resource manager with kerberos authentication kylin.job.status.with.kerberos=false + ## kylin security configurations # spring security profile, options: testing, ldap, saml http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f4b359f..76961c5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -652,14 +652,6 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.init.tasks"); } - public String getMRBatchEngineV1Class() { - return getOptional("kylin.cube.mr.engine.v1.class", "org.apache.kylin.engine.mr.MRBatchCubingEngine"); - } - - public String getMRBatchEngineV2Class() { - return getOptional("kylin.cube.mr.engine.v2.class", "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); - } - public int getDimCountDistinctMaxCardinality() { return Integer.parseInt(getOptional("kylin.query.dim.distinct.max", "5000000")); } @@ -668,4 +660,24 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.job.cubing.inmem.sampling.hll.precision", "14")); } + public Map<Integer, String> getJobEngines() { + return convertKeyToInteger(getPropertiesByPrefix("kylin.job.engine.")); + } + + public Map<Integer, String> getSourceEngines() { + return convertKeyToInteger(getPropertiesByPrefix("kylin.source.engine.")); + } + + public Map<Integer, String> getStorageEngines() { + return convertKeyToInteger(getPropertiesByPrefix("kylin.storage.engine.")); + } + + private Map<Integer, String> convertKeyToInteger(Map<String, String> map) { + Map<Integer, String> result = Maps.newLinkedHashMap(); + for (Entry<String, String> entry : map.entrySet()) { + result.put(Integer.valueOf(entry.getKey()), entry.getValue()); + } + return result; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java index bba9060..79d2f81 100644 --- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java +++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java @@ -18,10 +18,6 @@ package org.apache.kylin.engine; -import static org.apache.kylin.metadata.model.IEngineAware.ID_MR_V1; -import static org.apache.kylin.metadata.model.IEngineAware.ID_MR_V2; - -import java.util.HashMap; import java.util.Map; import org.apache.kylin.common.KylinConfig; @@ -35,9 +31,7 @@ public class EngineFactory { private static ImplementationSwitch<IBatchCubingEngine> batchEngines; private static ImplementationSwitch<IStreamingCubingEngine> streamingEngines; static { - Map<Integer, String> impls = new HashMap<>(); - impls.put(ID_MR_V1, KylinConfig.getInstanceFromEnv().getMRBatchEngineV1Class()); - impls.put(ID_MR_V2, KylinConfig.getInstanceFromEnv().getMRBatchEngineV2Class()); + Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getJobEngines(); batchEngines = new ImplementationSwitch<IBatchCubingEngine>(impls, IBatchCubingEngine.class); impls.clear(); http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java index 882b2e3..a9fb491 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IEngineAware.java @@ -24,6 +24,6 @@ public interface IEngineAware { public static final int ID_MR_V2 = 2; public static final int ID_MR_II = 3; public static final int ID_SPARK = 5; - + int getEngineType(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java index e552574..9f9edca 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java @@ -23,6 +23,6 @@ public interface IStorageAware { public static final int ID_HBASE = 0; public static final int ID_HYBRID = 1; public static final int ID_SHARDED_HBASE = 2; - + int getStorageType(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java b/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java index 54068f6..f701a0f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java @@ -18,11 +18,9 @@ package org.apache.kylin.source; -import static org.apache.kylin.metadata.model.ISourceAware.ID_HIVE; - -import java.util.HashMap; import java.util.Map; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ImplementationSwitch; import org.apache.kylin.metadata.model.ISourceAware; import org.apache.kylin.metadata.model.TableDesc; @@ -31,8 +29,7 @@ public class SourceFactory { private static ImplementationSwitch<ISource> sources; static { - Map<Integer, String> impls = new HashMap<>(); - impls.put(ID_HIVE, "org.apache.kylin.source.hive.HiveSource"); + Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getSourceEngines(); sources = new ImplementationSwitch<ISource>(impls, ISource.class); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java index da2f69c..e7dd53c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java @@ -18,13 +18,9 @@ package org.apache.kylin.storage; -import static org.apache.kylin.metadata.model.IStorageAware.ID_HBASE; -import static org.apache.kylin.metadata.model.IStorageAware.ID_HYBRID; -import static org.apache.kylin.metadata.model.IStorageAware.ID_SHARDED_HBASE; - -import java.util.HashMap; import java.util.Map; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ImplementationSwitch; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.realization.IRealization; @@ -35,10 +31,7 @@ public class StorageFactory { private static ImplementationSwitch<IStorage> storages; static { - Map<Integer, String> impls = new HashMap<>(); - impls.put(ID_HBASE, "org.apache.kylin.storage.hbase.HBaseStorage"); - impls.put(ID_SHARDED_HBASE, "org.apache.kylin.storage.hbase.HBaseStorage");//ID_SHARDED_HBASE is a special HBaseStorage - impls.put(ID_HYBRID, "org.apache.kylin.storage.hybrid.HybridStorage"); + Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getStorageEngines(); storages = new ImplementationSwitch<IStorage>(impls, IStorage.class); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 9ac7625..9605d76 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -15,6 +15,18 @@ # limitations under the License. # +# job engines +kylin.job.engine.0=org.apache.kylin.engine.mr.MRBatchCubingEngine +kylin.job.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2 + +# source engines +kylin.source.engine.0=org.apache.kylin.source.hive.HiveSource + +# storage engines +kylin.storage.engine.0=org.apache.kylin.storage.hbase.HBaseStorage +kylin.storage.engine.1=org.apache.kylin.storage.hybrid.HybridStorage +kylin.storage.engine.2=org.apache.kylin.storage.hbase.HBaseStorage + # optional information for the owner of kylin platform, it can be your team's email # currently it will be attached to each kylin's htable attribute kylin.owner=who...@kylin.apache.org http://git-wip-us.apache.org/repos/asf/kylin/blob/f31ad6e5/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index a004266..5247b02 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -15,6 +15,18 @@ # limitations under the License. # +# job engines +kylin.job.engine.0=org.apache.kylin.engine.mr.MRBatchCubingEngine +kylin.job.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2 + +# source engines +kylin.source.engine.0=org.apache.kylin.source.hive.HiveSource + +# storage engines +kylin.storage.engine.0=org.apache.kylin.storage.hbase.HBaseStorage +kylin.storage.engine.1=org.apache.kylin.storage.hybrid.HybridStorage +kylin.storage.engine.2=org.apache.kylin.storage.hbase.HBaseStorage + # kylin server's mode kylin.server.mode=all # optional information for the owner of kylin platform, it can be your team's email @@ -86,6 +98,7 @@ kylin.cube.algorithm=random kylin.security.profile=testing + ## Config for Restful APP ## # database connection settings: ldap.server=
