KYLIN-2921 Refactor DataModelDesc, split DataModelManager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f45d8133 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f45d8133 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f45d8133 Branch: refs/heads/master Commit: f45d8133b28f0063f1d73087f6dd65b34cf372cf Parents: 1f6e04c Author: Li Yang <[email protected]> Authored: Mon Oct 2 17:45:38 2017 +0800 Committer: Li Yang <[email protected]> Committed: Sun Oct 8 18:31:56 2017 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 10 +- .../streaming/StreamingTableDataGenerator.java | 4 +- .../apache/kylin/common/KylinConfigBase.java | 8 + .../java/org/apache/kylin/cube/CubeManager.java | 14 +- .../kylin/cube/cli/CubeSignatureRefresher.java | 12 +- .../org/apache/kylin/cube/model/CubeDesc.java | 4 +- .../kylin/cube/model/v1_4_0/CubeDesc.java | 4 +- .../upgrade/common/CubeMetadataUpgrade.java | 8 +- .../v1_4_0/CubeMetadataUpgrade_v_1_4_0.java | 11 +- .../kylin/cube/AggregationGroupRuleTest.java | 4 +- .../apache/kylin/cube/CubeManagerCacheTest.java | 4 +- .../kylin/cube/common/RowKeySplitterTest.java | 4 +- .../kylin/cube/cuboid/CuboidSchedulerTest.java | 4 +- .../apache/kylin/cube/cuboid/CuboidTest.java | 4 +- .../apache/kylin/cube/kv/RowKeyDecoderTest.java | 4 +- .../apache/kylin/cube/kv/RowKeyEncoderTest.java | 4 +- .../apache/kylin/dict/DictionaryManager.java | 16 +- .../apache/kylin/dict/lookup/SnapshotCLI.java | 4 +- .../kylin/dict/lookup/SnapshotManager.java | 12 +- .../kylin/dict/DictionaryManagerTest.java | 4 +- .../kylin/dict/lookup/LookupTableTest.java | 4 +- .../org/apache/kylin/job/dao/ExecutableDao.java | 4 +- .../apache/kylin/metadata/MetadataManager.java | 706 ------------------- .../kylin/metadata/TableMetadataManager.java | 494 +++++++++++++ .../metadata/filter/ColumnTupleFilter.java | 4 +- .../metadata/filter/UDF/MassInTupleFilter.java | 4 +- .../kylin/metadata/model/DataModelDesc.java | 125 +--- .../kylin/metadata/model/DataModelManager.java | 331 +++++++++ .../kylin/metadata/project/ProjectInstance.java | 7 - .../kylin/metadata/project/ProjectL2Cache.java | 10 +- .../kylin/metadata/project/ProjectManager.java | 29 +- .../kylin/source/datagen/ColumnGenConfig.java | 4 +- .../source/datagen/ModelDataGenerator.java | 4 +- .../kylin/metadata/MetadataManagerTest.java | 180 ----- .../metadata/TableMetadataManagerTest.java | 119 ++++ .../kylin/metadata/draft/DraftManagerTest.java | 4 +- .../kylin/metadata/model/DataModelDescTest.java | 11 +- .../metadata/model/DataModelManagerTest.java | 100 +++ .../kylin/metadata/model/JoinsTreeTest.java | 5 +- .../kylin/source/datagen/DataGenTest.java | 4 +- .../storage/gtrecord/CubeTupleConverter.java | 4 +- .../java/org/apache/kylin/engine/mr/MRUtil.java | 4 +- .../engine/mr/steps/MergeCuboidMapperTest.java | 4 +- .../localmeta/cube_desc/ci_inner_join_cube.json | 77 +- .../localmeta/cube_desc/ci_left_join_cube.json | 77 +- .../cube_desc/ut_inner_join_cube_partial.json | 41 +- .../model_desc/ci_inner_join_model.json | 59 +- .../model_desc/ci_left_join_model.json | 59 +- .../java/org/apache/kylin/query/H2Database.java | 4 +- .../apache/kylin/query/ITKylinQueryTest.java | 6 - .../source/hive/ITSnapshotManagerTest.java | 4 +- .../jdbc/ITJdbcSourceTableLoaderTest.java | 4 +- .../source/jdbc/ITJdbcTableReaderTest.java | 4 +- .../query/sql_computedcolumn/query01.sql | 14 - .../sql_computedcolumn/query01.sql.compare | 14 - .../query/sql_computedcolumn/query02.sql | 21 - .../sql_computedcolumn/query02.sql.compare | 21 - .../query/sql_computedcolumn/query03.sql | 21 - .../sql_computedcolumn/query03.sql.compare | 21 - .../query/sql_computedcolumn/query04.sql | 27 - .../sql_computedcolumn/query04.sql.compare | 26 - .../query/sql_verifyCount/query01.sql.expected | 2 +- .../query/sql_verifyCount/query02.sql.expected | 2 +- .../query/sql_verifyCount/query03.sql.expected | 2 +- .../query/sql_verifyCount/query04.sql.expected | 2 +- .../query/sql_verifyCount/query10.sql.expected | 2 +- .../query/sql_verifyCount/query11.sql.expected | 2 +- .../.settings/org.eclipse.core.resources.prefs | 1 + .../org/apache/kylin/query/QueryConnection.java | 7 +- .../apache/kylin/query/schema/OLAPSchema.java | 9 +- .../kylin/query/schema/OLAPSchemaFactory.java | 2 - .../query/security/QueryInterceptorUtil.java | 120 ---- .../kylin/rest/controller/ModelController.java | 6 +- .../rest/controller/ModelDescController.java | 4 +- .../apache/kylin/rest/job/HybridCubeCLI.java | 6 +- .../rest/response/DataModelDescResponse.java | 1 - .../kylin/rest/security/TableInterceptor.java | 17 +- .../apache/kylin/rest/service/BasicService.java | 11 +- .../apache/kylin/rest/service/CacheService.java | 5 - .../apache/kylin/rest/service/CubeService.java | 2 +- .../kylin/rest/service/ExtFilterService.java | 12 +- .../kylin/rest/service/HybridService.java | 2 +- .../apache/kylin/rest/service/ModelService.java | 116 +-- .../rest/service/TableSchemaUpdateChecker.java | 6 +- .../apache/kylin/rest/service/TableService.java | 24 +- .../kylin/rest/service/CacheServiceTest.java | 37 +- .../kylin/rest/service/CubeServiceTest.java | 2 +- .../kylin/rest/service/JobServiceTest.java | 2 +- .../kylin/rest/service/ModelServiceTest.java | 77 +- .../kylin/rest/service/QueryServiceTest.java | 2 +- .../apache/kylin/source/hive/HiveMRInput.java | 4 +- .../kylin/source/hive/HiveMetadataExplorer.java | 4 +- .../cardinality/ColumnCardinalityMapper.java | 4 +- .../cardinality/HiveColumnCardinalityJob.java | 4 +- .../HiveColumnCardinalityUpdateJob.java | 4 +- .../kylin/storage/hbase/HBaseStorage.java | 4 +- .../hbase/util/ExtendCubeToHybridCLI.java | 6 +- .../cube/MeasureTypeOnlyAggrInBaseTest.java | 4 +- .../hbase/steps/RowValueDecoderTest.java | 4 +- .../apache/kylin/tool/CubeMetaExtractor.java | 6 +- .../org/apache/kylin/tool/CubeMetaIngester.java | 38 +- .../kylin/tool/ExtendCubeToHybridCLI.java | 6 +- .../apache/kylin/tool/KylinLogExtractor.java | 6 +- .../apache/kylin/tool/CubeMetaIngesterTest.java | 6 +- 104 files changed, 1362 insertions(+), 2021 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index bdf7c47..5866c09 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -41,8 +41,9 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.job.streaming.StreamDataLoader; import org.apache.kylin.job.streaming.StreamingTableDataGenerator; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.source.ISampleDataDeployer; @@ -127,7 +128,7 @@ public class DeployUtil { System.out.println("build cube with random dataset"); // data is generated according to cube descriptor and saved in resource store - MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv()); ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc(modelName), 10000); gen.generate(); } else { @@ -190,8 +191,9 @@ public class DeployUtil { private static void deployTables(String modelName) throws Exception { - MetadataManager metaMgr = MetadataManager.getInstance(config()); - DataModelDesc model = metaMgr.getDataModelDesc(modelName); + TableMetadataManager metaMgr = TableMetadataManager.getInstance(config()); + DataModelManager modelMgr = DataModelManager.getInstance(config()); + DataModelDesc model = modelMgr.getDataModelDesc(modelName); Set<TableRef> tables = model.getAllTables(); Set<String> TABLE_NAMES = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java index 2144c3a..1fef07e 100644 --- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java +++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Random; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; @@ -51,7 +51,7 @@ public class StreamingTableDataGenerator { Preconditions.checkArgument(recordCount > 0); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName, prj); + TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, prj); SortedMultiset<Long> times = TreeMultiset.create(); Random r = new Random(); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index ee05d69..f362884 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -275,7 +275,15 @@ abstract public class KylinConfigBase implements Serializable { r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html return r; } + + public String getDataModelImpl() { + return getOptional("kylin.metadata.data-model-impl", null); + } + public String getDataModelManagerImpl() { + return getOptional("kylin.metadata.data-model-manager-impl", null); + } + public String[] getRealizationProviders() { return getOptionalStringArray("kylin.metadata.realization-providers", // new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 33dc312..3bb9f21 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -51,7 +51,7 @@ import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.dict.lookup.LookupStringTable; import org.apache.kylin.dict.lookup.SnapshotManager; import org.apache.kylin.dict.lookup.SnapshotTable; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; @@ -183,7 +183,6 @@ public class CubeManager implements IRealizationProvider { } public CubeInstance getCube(String cubeName) { - cubeName = cubeName.toUpperCase(); return cubeMap.get(cubeName); } @@ -205,7 +204,6 @@ public class CubeManager implements IRealizationProvider { */ public List<CubeInstance> getCubesByDesc(String descName) { - descName = descName.toUpperCase(); List<CubeInstance> list = listAllCubes(); List<CubeInstance> result = new ArrayList<CubeInstance>(); Iterator<CubeInstance> it = list.iterator(); @@ -279,7 +277,7 @@ public class CubeManager implements IRealizationProvider { } public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException { - MetadataManager metaMgr = getMetadataManager(); + TableMetadataManager metaMgr = getTableManager(); SnapshotManager snapshotMgr = getSnapshotManager(); TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, cubeSeg.getProject())); @@ -644,7 +642,7 @@ public class CubeManager implements IRealizationProvider { try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); - TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); + TableDesc tableDesc = getTableManager().getTableDesc(tableName, cubeSegment.getProject()); return new LookupStringTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException( @@ -808,10 +806,10 @@ public class CubeManager implements IRealizationProvider { && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming")); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(config); + private TableMetadataManager getTableManager() { + return TableMetadataManager.getInstance(config); } - + private DictionaryManager getDictionaryManager() { return DictionaryManager.getInstance(config); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java index 1db804f..d07c93b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java @@ -18,6 +18,7 @@ package org.apache.kylin.cube.cli; +import java.io.IOException; import java.util.List; import org.apache.commons.lang.ArrayUtils; @@ -26,7 +27,8 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.cachesync.Broadcaster; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.project.ProjectManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,8 +77,12 @@ public class CubeSignatureRefresher { } private void verify() { - MetadataManager.getInstance(config).reload(); - CubeDescManager.clearCache(); + try { + Broadcaster.getInstance(config).notifyClearAll(); + } catch (IOException e) { + throw new RuntimeException(e); + } + DataModelManager.getInstance(config); CubeDescManager.getInstance(config); CubeManager.getInstance(config); ProjectManager.getInstance(config); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 827ac81..6a75083 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -56,7 +56,6 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; @@ -65,6 +64,7 @@ import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectInstance; @@ -583,7 +583,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", this.rowkey.getRowKeyColumns().length); - this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName); + this.model = DataModelManager.getInstance(config).getDataModelDesc(modelName); checkNotNull(this.model, "DateModelDesc(%s) not found", modelName); for (DimensionDesc dim : dimensions) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java index f1a57e0..f5c4a0d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java @@ -43,13 +43,13 @@ import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.CaseInsensitiveStringMap; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -454,7 +454,7 @@ public class CubeDesc extends RootPersistentEntity { this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); } - this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); + this.model = DataModelManager.getInstance(config).getDataModelDesc(this.modelName); if (this.model == null) { this.addError("No data model found with name '" + modelName + "'."); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java index f70286d..f2f6b57 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java @@ -29,7 +29,7 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.project.ProjectManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,7 +63,7 @@ public abstract class CubeMetadataUpgrade { } public void clear() { - MetadataManager.clearCache(); + DataModelManager.clearCache(); CubeDescManager.clearCache(); CubeManager.clearCache(); ProjectManager.clearCache(); @@ -73,8 +73,8 @@ public abstract class CubeMetadataUpgrade { logger.info("================================================================="); logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:"); logger.info("================================================================="); - MetadataManager.clearCache(); - MetadataManager.getInstance(config); + DataModelManager.clearCache(); + DataModelManager.getInstance(config); CubeDescManager.clearCache(); CubeDescManager.getInstance(config); CubeManager.clearCache(); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java index 29465f7..1e90910 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java @@ -32,8 +32,9 @@ import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.cube.model.v1_4_0.CubeDesc; import org.apache.kylin.cube.model.v1_4_0.DimensionDesc; import org.apache.kylin.cube.upgrade.common.CubeMetadataUpgrade; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ModelDimensionDesc; import org.apache.kylin.metadata.model.PartitionDesc; @@ -82,8 +83,8 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade { } private DataModelDesc getDataModelDesc(String modelName) { - MetadataManager.clearCache(); - return MetadataManager.getInstance(config).getDataModelDesc(modelName); + DataModelManager.clearCache(); + return DataModelManager.getInstance(config).getDataModelDesc(modelName); } public void dowork() { @@ -91,7 +92,7 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade { for (String path : paths) { logger.info("CubeMetadataUpgrade_v_1_4_0 handling in dowork {}", path); CubeDesc cubeDesc = loadOldCubeDesc(path); - cubeDesc.init(config, MetadataManager.getInstance(config).getAllTablesMap(cubeDesc.getProject())); + cubeDesc.init(config, TableMetadataManager.getInstance(config).getAllTablesMap(cubeDesc.getProject())); upgradeDataModelDesc(cubeDesc); upgradeCubeDesc(cubeDesc); @@ -144,7 +145,7 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade { } if (upgrade) { - store.putResource(modelDesc.getResourcePath(), modelDesc, MetadataManager.MODELDESC_SERIALIZER); + store.putResource(modelDesc.getResourcePath(), modelDesc, DataModelManager.getInstance(config).getDataModelSerializer()); updatedResources.add(modelDesc.getResourcePath()); } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java index 62b9c98..1444ee1 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java @@ -32,7 +32,7 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.validation.IValidatorRule; import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -41,7 +41,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java index de3048f..52b9042 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.junit.After; @@ -41,7 +41,7 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); CubeManager.clearCache(); ProjectManager.clearCache(); cubeManager = CubeManager.getInstance(getTestConfig()); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java index b41f23b..6d32586 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -33,7 +33,7 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java index 3f3a7c4..09200b8 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java @@ -31,7 +31,7 @@ import java.util.Set; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -47,7 +47,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java index 0fa849e..0a77bdc 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -61,7 +61,7 @@ public class CuboidTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java index ec1f221..459e734 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java @@ -29,7 +29,7 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -40,7 +40,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java index 5af8d8a..dcd883e 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java @@ -29,7 +29,7 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -40,7 +40,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - MetadataManager.clearCache(); + DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index 0eab8e4..e97899c 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -34,8 +34,8 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.source.IReadableTable; import org.apache.kylin.source.IReadableTable.TableSignature; @@ -176,7 +176,7 @@ public class DictionaryManager { } private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<String> dict) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); + ResourceStore store = DataModelManager.getInstance(config).getStore(); NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; @@ -343,7 +343,7 @@ public class DictionaryManager { } private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { - final ResourceStore store = MetadataManager.getInstance(config).getStore(); + final ResourceStore store = DataModelManager.getInstance(config).getStore(); final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); TableSignature input = dictInfo.getInput(); @@ -357,7 +357,7 @@ public class DictionaryManager { } private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException { - final ResourceStore store = MetadataManager.getInstance(config).getStore(); + final ResourceStore store = DataModelManager.getInstance(config).getStore(); final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); DictionaryInfo largestDict = null; @@ -376,7 +376,7 @@ public class DictionaryManager { public void removeDictionary(String resourcePath) throws IOException { logger.info("Remvoing dict: " + resourcePath); - ResourceStore store = MetadataManager.getInstance(config).getStore(); + ResourceStore store = DataModelManager.getInstance(config).getStore(); store.deleteResource(resourcePath); dictCache.invalidate(resourcePath); } @@ -386,7 +386,7 @@ public class DictionaryManager { info.setSourceTable(srcTable); info.setSourceColumn(srcCol); - ResourceStore store = MetadataManager.getInstance(config).getStore(); + ResourceStore store = DataModelManager.getInstance(config).getStore(); NavigableSet<String> existings = store.listResources(info.getResourceDir()); if (existings == null) return; @@ -396,7 +396,7 @@ public class DictionaryManager { } void save(DictionaryInfo dict) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); + ResourceStore store = DataModelManager.getInstance(config).getStore(); String path = dict.getResourcePath(); logger.info("Saving dictionary at " + path); @@ -412,7 +412,7 @@ public class DictionaryManager { } DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); + ResourceStore store = DataModelManager.getInstance(config).getStore(); logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" + loadDictObj + ") at " + resourcePath); DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java index 0fe102b..2093d23 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java @@ -21,7 +21,7 @@ package org.apache.kylin.dict.lookup; import java.io.IOException; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.source.SourceFactory; @@ -35,7 +35,7 @@ public class SnapshotCLI { private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); - MetadataManager metaMgr = MetadataManager.getInstance(conf); + TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java index b997a88..c10deb4 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java @@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.source.IReadableTable; import org.apache.kylin.source.IReadableTable.TableSignature; @@ -116,7 +116,7 @@ public class SnapshotManager { } public void removeSnapshot(String resourcePath) throws IOException { - ResourceStore store = MetadataManager.getInstance(this.config).getStore(); + ResourceStore store = DataModelManager.getInstance(this.config).getStore(); store.deleteResource(resourcePath); snapshotCache.invalidate(resourcePath); } @@ -171,7 +171,7 @@ public class SnapshotManager { } private String checkDupByInfo(SnapshotTable snapshot) throws IOException { - ResourceStore store = MetadataManager.getInstance(this.config).getStore(); + ResourceStore store = DataModelManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) @@ -189,7 +189,7 @@ public class SnapshotManager { } private String checkDupByContent(SnapshotTable snapshot) throws IOException { - ResourceStore store = MetadataManager.getInstance(this.config).getStore(); + ResourceStore store = DataModelManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) @@ -205,14 +205,14 @@ public class SnapshotManager { } private void save(SnapshotTable snapshot) throws IOException { - ResourceStore store = MetadataManager.getInstance(this.config).getStore(); + ResourceStore store = DataModelManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.putResource(path, snapshot, SnapshotTableSerializer.FULL_SERIALIZER); } private SnapshotTable load(String resourcePath, boolean loadData) throws IOException { logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData); - ResourceStore store = MetadataManager.getInstance(this.config).getStore(); + ResourceStore store = DataModelManager.getInstance(this.config).getStore(); SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java index 9c126b4..6a86e33 100755 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java @@ -26,8 +26,8 @@ import java.io.IOException; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TblColRef; import org.junit.After; import org.junit.Before; @@ -50,7 +50,7 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase { public void testBuildSaveDictionary() throws IOException { KylinConfig config = KylinConfig.getInstanceFromEnv(); DictionaryManager dictMgr = DictionaryManager.getInstance(config); - MetadataManager metaMgr = MetadataManager.getInstance(config); + DataModelManager metaMgr = DataModelManager.getInstance(config); DataModelDesc model = metaMgr.getDataModelDesc("test_kylin_inner_join_model_desc"); TblColRef col = model.findColumn("lstg_format_name"); http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java index 57d70cf..f53e8fe 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java @@ -28,7 +28,7 @@ import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.common.util.Pair; import org.apache.kylin.dict.TrieDictionaryForest; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.model.TableDesc; import org.junit.After; import org.junit.Assert; @@ -121,7 +121,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase { public LookupTable<String> initLookupTable() throws Exception { - MetadataManager metaMgr = MetadataManager.getInstance(config); + TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); String tableName = "EDW.TEST_CAL_DT"; String[] pkCols = new String[] { "CAL_DT" }; http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java index bd020f8..16875b1 100644 --- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java +++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java @@ -31,7 +31,7 @@ import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.job.exception.PersistentException; -import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,7 +73,7 @@ public class ExecutableDao { private ExecutableDao(KylinConfig config) { logger.info("Using metadata url: " + config); - this.store = MetadataManager.getInstance(config).getStore(); + this.store = DataModelManager.getInstance(config).getStore(); } private String pathOfJob(ExecutablePO job) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java deleted file mode 100644 index bfc7f08..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ /dev/null @@ -1,706 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.metadata; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import javax.annotation.Nullable; - -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.RawResource; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.metadata.cachesync.Broadcaster; -import org.apache.kylin.metadata.cachesync.Broadcaster.Event; -import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.ExternalFilterDesc; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TableExtDesc; -import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.metadata.project.ProjectManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Function; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -/** - * Serves (and caches) metadata for Kylin instance. - * <p/> - * Also provides a ResourceStore for general purpose data persistence. - * Metadata is serialized as JSON and stored in ResourceStore. - * - * @author yangli9 - */ -public class MetadataManager { - - private static final Logger logger = LoggerFactory.getLogger(MetadataManager.class); - - public static final Serializer<TableDesc> TABLE_SERIALIZER = new JsonSerializer<TableDesc>(TableDesc.class); - public static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<TableExtDesc>( - TableExtDesc.class); - public static final Serializer<DataModelDesc> MODELDESC_SERIALIZER = new JsonSerializer<DataModelDesc>( - DataModelDesc.class); - public static final Serializer<ExternalFilterDesc> EXTERNAL_FILTER_DESC_SERIALIZER = new JsonSerializer<ExternalFilterDesc>( - ExternalFilterDesc.class); - - // static cached instances - private static final ConcurrentMap<KylinConfig, MetadataManager> CACHE = new ConcurrentHashMap<KylinConfig, MetadataManager>(); - - public static MetadataManager getInstance(KylinConfig config) { - MetadataManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (MetadataManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new MetadataManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist, current keys: {}", StringUtils - .join(Iterators.transform(CACHE.keySet().iterator(), new Function<KylinConfig, String>() { - @Nullable - @Override - public String apply(@Nullable KylinConfig input) { - return String.valueOf(System.identityHashCode(input)); - } - }), ",")); - } - - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init MetadataManager from " + config, e); - } - } - } - - public static void clearCache() { - CACHE.clear(); - } - - // ============================================================================ - - private KylinConfig config; - // table name ==> SourceTable - private CaseInsensitiveStringCache<TableDesc> srcTableMap; - // name => value - private CaseInsensitiveStringCache<TableExtDesc> srcTableExtMap; - // name => DataModelDesc - private CaseInsensitiveStringCache<DataModelDesc> dataModelDescMap; - // name => External Filter Desc - private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap; - - private MetadataManager(KylinConfig config) throws IOException { - init(config); - } - - /** - * Tell MetadataManager that the instance has changed. The cube info will - * be stored Reload the cube desc and source table A broadcast must be sent - * out - * - * @return - * @throws IOException - */ - public void reload() { - clearCache(); - getInstance(config); - } - - public KylinConfig getConfig() { - return config; - } - - public ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - public List<DataModelDesc> listDataModels() { - return Lists.newArrayList(this.dataModelDescMap.values()); - } - - public List<TableDesc> listAllTables(String prj) { - return Lists.newArrayList(getAllTablesMap(prj).values()); - } - - public List<ExternalFilterDesc> listAllExternalFilters() { - return Lists.newArrayList(extFilterMap.values()); - } - - public Map<String, TableDesc> getAllTablesMap(String prj) { - Map<String, TableDesc> globalTables = new LinkedHashMap<>(); - Map<String, TableDesc> projectTables = new LinkedHashMap<>(); - - for (TableDesc t : srcTableMap.values()) { - if (t.getProject() == null) - globalTables.put(t.getIdentity(), t); - else if (t.getProject().equals(prj)) - projectTables.put(t.getIdentity(), t); - } - - Map<String, TableDesc> result = globalTables; - result.putAll(projectTables); - return result; - } - - /** - * Get TableDesc by name - */ - public TableDesc getTableDesc(String tableName, String prj) { - if (tableName.indexOf(".") < 0) - tableName = "DEFAULT." + tableName; - - tableName.toUpperCase(); - - TableDesc result = srcTableMap.get(mapKey(tableName, prj)); - if (result == null) - result = srcTableMap.get(mapKey(tableName, null)); - - return result; - } - - public ExternalFilterDesc getExtFilterDesc(String filterTableName) { - ExternalFilterDesc result = extFilterMap.get(filterTableName); - return result; - } - - /** - * Get table extended info. Keys are defined in {@link MetadataConstants} - * - * @param tableName - * @return - */ - public TableExtDesc getTableExt(String tableName, String prj) { - TableDesc t = getTableDesc(tableName, prj); - if (t == null) - return null; - - return getTableExt(t); - } - - public TableExtDesc getTableExt(TableDesc t) { - TableExtDesc result = srcTableExtMap.get(mapKey(t.getIdentity(), t.getProject())); - - // avoid returning null, since the TableDesc exists - if (null == result) { - result = new TableExtDesc(); - result.setIdentity(t.getIdentity()); - result.setUuid(UUID.randomUUID().toString()); - result.setLastModified(0); - result.init(t.getProject()); - srcTableExtMap.put(mapKey(t.getIdentity(), t.getProject()), result); - } - return result; - } - - public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { - if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { - throw new IllegalArgumentException(); - } - - // updating a legacy global table - if (tableExt.getProject() == null) { - if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) - throw new IllegalStateException( - "Updating a legacy global TableExtDesc while a project level version exists: " - + tableExt.getIdentity() + ", " + prj); - prj = tableExt.getProject(); - } - - tableExt.init(prj); - - String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); - - ResourceStore store = getStore(); - - TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); - if (t != null && t.getIdentity() == null) - store.deleteResource(path); - - store.putResource(path, tableExt, TABLE_EXT_SERIALIZER); - srcTableExtMap.put(mapKey(tableExt.getIdentity(), tableExt.getProject()), tableExt); - } - - public void removeTableExt(String tableName, String prj) throws IOException { - // note, here assume always delete TableExtDesc first, then TableDesc - TableExtDesc t = getTableExt(tableName, prj); - if (t == null) - return; - - String path = TableExtDesc.concatResourcePath(t.getIdentity(), t.getProject()); - getStore().deleteResource(path); - srcTableExtMap.remove(mapKey(t.getIdentity(), t.getProject())); - } - - public void saveSourceTable(TableDesc srcTable, String prj) throws IOException { - if (srcTable.getUuid() == null || srcTable.getIdentity() == null) { - throw new IllegalArgumentException(); - } - - srcTable.init(prj); - - String path = TableDesc.concatResourcePath(srcTable.getIdentity(), prj); - getStore().putResource(path, srcTable, TABLE_SERIALIZER); - - srcTableMap.put(mapKey(srcTable.getIdentity(), prj), srcTable); - } - - public void removeSourceTable(String tableIdentity, String prj) throws IOException { - TableDesc t = getTableDesc(tableIdentity, prj); - if (t == null) - return; - - String path = TableDesc.concatResourcePath(t.getIdentity(), t.getProject()); - getStore().deleteResource(path); - srcTableMap.remove(mapKey(t.getIdentity(), t.getProject())); - } - - public void saveExternalFilter(ExternalFilterDesc desc) throws IOException { - if (desc.getUuid() == null) { - throw new IllegalArgumentException("UUID not set."); - } - String path = desc.getResourcePath(); - getStore().putResource(path, desc, EXTERNAL_FILTER_DESC_SERIALIZER); - desc = reloadExternalFilterAt(path); - extFilterMap.put(desc.getName(), desc); - - } - - public void removeExternalFilter(String name) throws IOException { - String path = ExternalFilterDesc.concatResourcePath(name); - getStore().deleteResource(path); - extFilterMap.remove(name); - - } - - private void init(KylinConfig config) throws IOException { - this.config = config; - this.srcTableMap = new CaseInsensitiveStringCache<>(config, "table"); - this.srcTableExtMap = new CaseInsensitiveStringCache<>(config, "table_ext"); - this.dataModelDescMap = new CaseInsensitiveStringCache<>(config, "data_model"); - this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter"); - - reloadAllSourceTable(); - reloadAllTableExt(); - reloadAllDataModel(); - reloadAllExternalFilter(); - - // touch lower level metadata before registering my listener - Broadcaster.getInstance(config).registerListener(new SrcTableSyncListener(), "table"); - Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext"); - Broadcaster.getInstance(config).registerListener(new DataModelSyncListener(), "data_model"); - Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter"); - } - - private class SrcTableSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - srcTableMap.removeLocal(cacheKey); - else - reloadSourceTableAt(TableDesc.concatRawResourcePath(cacheKey)); - - Pair<String, String> pair = TableDesc.parseResourcePath(cacheKey); - String table = pair.getFirst(); - String prj = pair.getSecond(); - - if (prj == null) { - for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) { - broadcaster.notifyProjectSchemaUpdate(p.getName()); - } - } else { - broadcaster.notifyProjectSchemaUpdate(prj); - } - } - } - - private class SrcTableExtSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - srcTableExtMap.removeLocal(cacheKey); - else - reloadTableExtAt(TableExtDesc.concatRawResourcePath(cacheKey)); - } - } - - private class DataModelSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - - @Override - public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { - for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) { - reloadDataModelDescAt(DataModelDesc.concatResourcePath(model)); - } - } - - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - dataModelDescMap.removeLocal(cacheKey); - else - reloadDataModelDescAt(DataModelDesc.concatResourcePath(cacheKey)); - - for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(cacheKey)) { - broadcaster.notifyProjectSchemaUpdate(prj.getName()); - } - } - } - - private class ExtFilterSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - extFilterMap.removeLocal(cacheKey); - else - reloadExtFilter(cacheKey); - } - } - - private void reloadAllTableExt() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading Table_exd info from folder " - + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT)); - - srcTableExtMap.clear(); - - List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadTableExtAt(path); - } - - logger.debug("Loaded " + srcTableExtMap.size() + " SourceTable EXD(s)"); - } - - private TableExtDesc reloadTableExtAt(String path) throws IOException { - ResourceStore store = getStore(); - String prj = TableExtDesc.parseResourcePath(path).getSecond(); - - TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); - - if (t == null) { - return null; - } - - // convert old tableExt json to new one - if (t.getIdentity() == null) { - t = convertOldTableExtToNewer(path); - } - - t.init(prj); - - srcTableExtMap.putLocal(mapKey(t.getIdentity(), prj), t); - return t; - } - - private String mapKey(String identity, String prj) { - return prj == null ? identity : identity + "--" + prj; - } - - private TableExtDesc convertOldTableExtToNewer(String path) throws IOException { - Map<String, String> attrs = Maps.newHashMap(); - - ResourceStore store = getStore(); - RawResource res = store.getResource(path); - - InputStream is = res.inputStream; - - try { - attrs.putAll(JsonUtil.readValue(is, HashMap.class)); - } finally { - if (is != null) - is.close(); - } - - String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY); - - // parse table identity from file name - String file = path; - if (file.indexOf("/") > -1) { - file = file.substring(file.lastIndexOf("/") + 1); - } - String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase(); - TableExtDesc result = new TableExtDesc(); - result.setIdentity(tableIdentity); - result.setUuid(UUID.randomUUID().toString()); - result.setLastModified(0); - result.setCardinality(cardinality); - return result; - } - - private void reloadAllExternalFilter() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading ExternalFilter from folder " - + store.getReadableResourcePath(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT)); - - extFilterMap.clear(); - - List<String> paths = store.collectResourceRecursively(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadExternalFilterAt(path); - } - - logger.debug("Loaded " + extFilterMap.size() + " ExternalFilter(s)"); - } - - private void reloadAllSourceTable() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading SourceTable from folder " - + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT)); - - srcTableMap.clear(); - - List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadSourceTableAt(path); - } - - logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)"); - } - - private TableDesc reloadSourceTableAt(String path) throws IOException { - ResourceStore store = getStore(); - String prj = TableDesc.parseResourcePath(path).getSecond(); - - TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER); - if (t == null) { - return null; - } - t.init(prj); - - srcTableMap.putLocal(mapKey(t.getIdentity(), prj), t); - - return t; - } - - private ExternalFilterDesc reloadExternalFilterAt(String path) throws IOException { - ResourceStore store = getStore(); - ExternalFilterDesc t = store.getResource(path, ExternalFilterDesc.class, EXTERNAL_FILTER_DESC_SERIALIZER); - if (t == null) { - return null; - } - extFilterMap.putLocal(t.getName(), t); - - return t; - } - - public void reloadExtFilter(String extFilterName) throws IOException { - reloadExternalFilterAt(ExternalFilterDesc.concatResourcePath(extFilterName)); - } - - public DataModelDesc getDataModelDesc(String name) { - return dataModelDescMap.get(name); - } - - public List<DataModelDesc> getModels() { - return new ArrayList<>(dataModelDescMap.values()); - } - - public List<DataModelDesc> getModels(String projectName) { - ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName); - ArrayList<DataModelDesc> ret = new ArrayList<>(); - - if (projectInstance != null && projectInstance.getModels() != null) { - for (String modelName : projectInstance.getModels()) { - DataModelDesc model = getDataModelDesc(modelName); - if (null != model) { - ret.add(model); - } else { - logger.error("Failed to load model " + modelName); - } - } - } - - return ret; - } - - // within a project, find models that use the specified table - public List<String> getModelsUsingTable(TableDesc table, String project) throws IOException { - List<String> models = new ArrayList<>(); - for (DataModelDesc modelDesc : getModels(project)) { - if (modelDesc.containsTable(table)) - models.add(modelDesc.getName()); - } - return models; - } - - public boolean isTableInAnyModel(TableDesc table) { - for (DataModelDesc modelDesc : getModels()) { - if (modelDesc.containsTable(table)) - return true; - } - return false; - } - - private void reloadAllDataModel() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading DataModel from folder " - + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT)); - - dataModelDescMap.clear(); - - List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - - try { - logger.info("Reloading data model at " + path); - reloadDataModelDescAt(path); - } catch (IllegalStateException e) { - logger.error("Error to load DataModel at " + path, e); - continue; - } - } - - logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)"); - } - - public DataModelDesc reloadDataModelDescAt(String path) { - ResourceStore store = getStore(); - try { - DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER); - String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName(); - - if (!dataModelDesc.isDraft()) - dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels()); - - dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc); - return dataModelDesc; - } catch (Exception e) { - throw new IllegalStateException("Error to load " + path, e); - } - } - - // sync on update - public DataModelDesc dropModel(DataModelDesc desc) throws IOException { - logger.info("Dropping model '" + desc.getName() + "'"); - ResourceStore store = getStore(); - store.deleteResource(desc.getResourcePath()); - // delete model from project - ProjectManager.getInstance(config).removeModelFromProjects(desc.getName()); - // clean model cache - this.afterModelDropped(desc); - return desc; - } - - private void afterModelDropped(DataModelDesc desc) { - removeModelCache(desc.getName()); - } - - public void removeModelCache(String modelName) { - dataModelDescMap.remove(modelName); - } - - public DataModelDesc createDataModelDesc(DataModelDesc desc, String projectName, String owner) throws IOException { - String name = desc.getName(); - if (dataModelDescMap.containsKey(name)) - throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists"); - - ProjectManager prjMgr = ProjectManager.getInstance(config); - ProjectInstance prj = prjMgr.getProject(projectName); - if (prj.containsModel(name)) - throw new IllegalStateException("project " + projectName + " already contains model " + name); - - try { - // Temporarily register model under project, because we want to - // update project formally after model is saved. - prj.getModels().add(name); - - desc.setOwner(owner); - desc = saveDataModelDesc(desc); - - } finally { - prj.getModels().remove(name); - } - - // now that model is saved, update project formally - prjMgr.updateModelToProject(name, projectName); - - return desc; - } - - public DataModelDesc updateDataModelDesc(DataModelDesc desc) throws IOException { - String name = desc.getName(); - if (!dataModelDescMap.containsKey(name)) { - throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist."); - } - - return saveDataModelDesc(desc); - } - - private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException { - - String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName(); - - if (!dataModelDesc.isDraft()) - dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels()); - - String path = dataModelDesc.getResourcePath(); - getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER); - dataModelDescMap.put(dataModelDesc.getName(), dataModelDesc); - - return dataModelDesc; - } -}
