KYLIN-2294 Refactor CI, reduce cube number to 3
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0fbffd5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0fbffd5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0fbffd5 Branch: refs/heads/master Commit: f0fbffd58a37aa9de22b887e385502b7baf670ea Parents: 9dbc24b Author: Li Yang <liy...@apache.org> Authored: Mon Dec 19 18:03:37 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Thu Dec 22 10:38:15 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 9 +- .../apache/kylin/common/KylinConfigBase.java | 5 - .../kylin/common/persistence/ResourceStore.java | 2 +- .../org/apache/kylin/cube/CubeDescTest.java | 8 + .../validation/rule/DictionaryRuleTest.java | 6 +- .../kylin/cube/project/ProjectManagerTest.java | 4 +- .../kylin/source/datagen/ColumnGenerator.java | 14 +- .../source/datagen/ModelDataGenerator.java | 31 +- .../kylin/measure/hllc/HLLCounterOldTest.java | 2 +- .../kylin/measure/hllc/HLLCounterTest.java | 2 +- .../kylin/metadata/MetadataManagerTest.java | 7 + .../mr/steps/FactDistinctColumnsReducer.java | 3 + .../localmeta/cube/ci_inner_join_cube.json | 10 + .../localmeta/cube/ci_left_join_cube.json | 10 + .../localmeta/cube_desc/ci_inner_join_cube.json | 286 +++++++++++++++++++ .../localmeta/cube_desc/ci_left_join_cube.json | 286 +++++++++++++++++++ ...t_kylin_cube_without_slr_left_join_desc.json | 4 +- .../localmeta/hybrid/ci_inner_join_hybrid.json | 14 + .../hybrid/test_kylin_hybrid_inner_join.json | 14 - .../model_desc/ci_inner_join_model.json | 139 +++++++++ .../model_desc/ci_left_join_model.json | 141 +++++++++ .../test_kylin_inner_join_model_desc.json | 7 +- .../test_kylin_left_join_model_desc.json | 7 +- .../localmeta/project/default.json | 92 +++--- .../table/DEFAULT.TEST_KYLIN_FACT.json | 39 ++- .../localmeta/table/DEFAULT.TEST_ORDER.json | 18 ++ .../sandbox/kylin_job_conf_inmem.xml | 6 +- .../inmemcubing/ITDoggedCubeBuilderTest.java | 4 +- .../inmemcubing/ITInMemCubeBuilderTest.java | 2 +- .../kylin/provision/BuildCubeWithEngine.java | 99 ++----- .../apache/kylin/query/ITCombinationTest.java | 15 +- .../apache/kylin/query/ITKylinQueryTest.java | 29 +- .../resources/query/sql_derived/query12.sql | 2 +- .../query/sql_distinct_precisely/query00.sql | 2 +- .../query/sql_distinct_precisely/query01.sql | 2 +- .../query/sql_distinct_precisely/query02.sql | 2 +- .../query/sql_distinct_precisely/query03.sql | 2 +- .../query/sql_distinct_precisely/query04.sql | 2 +- .../query/sql_intersect_count/query00.sql | 14 +- .../test/resources/query/sql_limit/query01.sql | 10 +- .../test/resources/query/sql_raw/query04.sql | 2 +- .../test/resources/query/sql_raw/query06.sql | 2 +- .../test/resources/query/sql_raw/query23.sql | 2 +- .../test/resources/query/sql_raw/query24.sql | 24 -- .../query/sql_raw/query26.sql.disabled | 24 -- .../resources/query/sql_tableau/query00.sql | 4 +- .../resources/query/sql_tableau/query10.sql | 2 + .../resources/query/sql_tableau/query11.sql | 2 + .../resources/query/sql_tableau/query12.sql | 2 + .../resources/query/sql_tableau/query13.sql | 2 + .../resources/query/sql_tableau/query14.sql | 7 +- .../resources/query/sql_tableau/query15.sql | 2 + .../resources/query/sql_tableau/query16.sql | 2 + .../resources/query/sql_tableau/query17.sql | 7 +- 54 files changed, 1136 insertions(+), 299 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index c8b0d43..564a4e2 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -124,13 +124,14 @@ public class DeployUtil { static final String TABLE_CAL_DT = "edw.test_cal_dt"; static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings"; static final String TABLE_KYLIN_FACT = "default.test_kylin_fact"; + static final String TABLE_ORDER = "default.test_order"; static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim"; static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table"; static final String TABLE_SITES = "edw.test_sites"; - static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES }; + static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES }; - public static void prepareTestDataForNormalCubes(String cubeName) throws Exception { + public static void prepareTestDataForNormalCubes(String modelName) throws Exception { boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData")); if (!buildCubeUsingProvidedData) { @@ -138,7 +139,7 @@ public class DeployUtil { // data is generated according to cube descriptor and saved in resource store MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - DataModelDesc model = mgr.getDataModelDesc("test_kylin_inner_join_model_desc"); + DataModelDesc model = mgr.getDataModelDesc(modelName); ModelDataGenerator gen = new ModelDataGenerator(model, 10000); gen.generate(); } else { @@ -230,6 +231,7 @@ public class DeployUtil { hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase()))); @@ -238,6 +240,7 @@ public class DeployUtil { hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_ORDER, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir)); hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir)); http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 01d1d36..11e48ac 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -496,11 +496,6 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column"); } - @Deprecated - public String getCreateFlatHiveTableMethod() { - return getOptional("kylin.source.hive.create-flat-table-method", "1"); - } - // ============================================================================ // STORAGE.HBASE // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 466d9d2..0580576 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -109,7 +109,7 @@ abstract public class ResourceStore { final protected KylinConfig kylinConfig; - public ResourceStore(KylinConfig kylinConfig) { + protected ResourceStore(KylinConfig kylinConfig) { this.kylinConfig = kylinConfig; } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 1daa748..fb003bc 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -41,6 +41,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * @author yangli9 @@ -91,6 +92,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase { } @Test + public void testCiCube() { + CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig()); + assertNotNull(mgr.getCubeDesc("ci_left_join_cube")); + assertNotNull(mgr.getCubeDesc("ci_inner_join_cube")); + } + + @Test public void testGoodInit() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.init(getTestConfig()); http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java index b6e0bcb..c272bb7 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java @@ -72,8 +72,8 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase { @Test public void testBadDesc() throws IOException { - testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("USER_ID", null, "FakeBuilderClass")); - testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("USER_ID", null, GlobalDictionaryBuilder.class.getName())); + testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("ORDER_ID", null, "FakeBuilderClass")); + testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("ORDER_ID", null, GlobalDictionaryBuilder.class.getName())); } @Test @@ -89,7 +89,7 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase { @Test public void testBadDesc4() throws IOException { testDictionaryDesc(ERROR_TRANSITIVE_REUSE, - DictionaryDesc.create("lstg_site_id", "USER_ID", null), + DictionaryDesc.create("lstg_site_id", "SELLER_ID", null), DictionaryDesc.create("price", "lstg_site_id", null)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java index 9728e52..a0fb4a3 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java @@ -147,7 +147,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase { ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig()); CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); - CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty"); + CubeInstance cube = cubeMgr.getCube("ci_left_join_cube"); assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); assertTrue(prjMgr.listAllRealizations("default").contains(cube)); @@ -162,7 +162,7 @@ public class ProjectManagerTest extends LocalFileMetadataTestCase { ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig()); CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); - CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty"); + CubeInstance cube = cubeMgr.getCube("ci_left_join_cube"); assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName()); http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java index 775f4fc..7c54714 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java @@ -250,8 +250,8 @@ public class ColumnGenerator { public String next() { if (values.isEmpty()) return null; - else - return values.get(rand.nextInt(values.size())); + + return values.get(rand.nextInt(values.size())); } } @@ -308,7 +308,15 @@ public class ColumnGenerator { @Override public String next() { - return rand.nextDouble() < nullPct || !input.hasNext() ? nullStr : input.next(); + String r = nullStr; + if (input.hasNext()) { + r = input.next(); + } + + if (rand.nextDouble() < nullPct) { + r = nullStr; + } + return r; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java index 60f2736..f1e6d9f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java @@ -21,6 +21,7 @@ package org.apache.kylin.source.datagen; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; @@ -34,8 +35,10 @@ import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; @@ -56,10 +59,14 @@ public class ModelDataGenerator { boolean outprint = false; // for debug public ModelDataGenerator(DataModelDesc model, int nRows) { - this(model, nRows, ResourceStore.getStore(model.getConfig()), "/data"); + this(model, nRows, ResourceStore.getStore(model.getConfig())); } - public ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) { + private ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore) { + this(model, nRows, outputStore, "/data"); + } + + private ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) { this.model = model; this.targetRows = nRows; this.outputStore = outputStore; @@ -268,8 +275,8 @@ public class ModelDataGenerator { } private void saveResource(byte[] content, String path) throws IOException { + System.out.println("Generated " + outputStore.getReadableResourcePath(path)); if (outprint) { - System.out.println("Generated " + path); System.out.println(Bytes.toString(content)); } outputStore.putResource(path, new ByteArrayInputStream(content), System.currentTimeMillis()); @@ -287,4 +294,22 @@ public class ModelDataGenerator { return model; } + public static void main(String[] args) throws IOException { + String modelName = args[0]; + int nRows = Integer.parseInt(args[1]); + String outputDir = args.length > 2 ? args[2] : null; + + KylinConfig conf = KylinConfig.getInstanceFromEnv(); + DataModelDesc model = MetadataManager.getInstance(conf).getDataModelDesc(modelName); + ResourceStore store = outputDir == null ? ResourceStore.getStore(conf) : ResourceStore.getStore(mockup(outputDir)); + + ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store); + gen.generate(); + } + + private static KylinConfig mockup(String outputDir) { + KylinConfig mockup = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv()); + mockup.setMetadataUrl(new File(outputDir).getAbsolutePath()); + return mockup; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java index c4a97cd..427d9a7 100644 --- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java @@ -157,7 +157,7 @@ public class HLLCounterOldTest { public void mergeTest() throws IOException { double error = 0; int n = 100; - for (int i = 0; i < n; i++) { + for (int i = 0; i < n; i += 10) { double e = merge(i); error += e; } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java index 26ad4a7..cd75eac 100644 --- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java @@ -82,7 +82,7 @@ public class HLLCounterTest { public void mergeTest() throws IOException { double error = 0; int n = 100; - for (int i = 0; i < n; i++) { + for (int i = 0; i < n; i += 10) { double e = merge(i); error += e; } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java index 204b4e0..a27f4cf 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java @@ -46,6 +46,13 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { public void after() throws Exception { this.cleanupTestMetadata(); } + + @Test + public void testCiModel() { + MetadataManager mgr = getInstance(getTestConfig()); + Assert.assertNotNull(mgr.getDataModelDesc("ci_left_join_model")); + Assert.assertNotNull(mgr.getDataModelDesc("ci_inner_join_model")); + } @Test public void testListAllTables() throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java index 3115fe4..0223914 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java @@ -128,6 +128,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK // local build dict isReducerLocalBuildDict = config.isReducerLocalBuildDict(); + if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder + isReducerLocalBuildDict = false; + } if (col != null && isReducerLocalBuildDict) { builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0); http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json new file mode 100644 index 0000000..f055be0 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json @@ -0,0 +1,10 @@ +{ + "uuid" : "8372c3b7-a33e-4b69-83dd-0bb8b1f8117e", + "last_modified" : 0, + "name" : "ci_inner_join_cube", + "owner" : null, + "descriptor" : "ci_inner_join_cube", + "segments" : [ ], + "status" : "DISABLED", + "create_time" : null +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube/ci_left_join_cube.json new file mode 100644 index 0000000..c3fddf0 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/ci_left_join_cube.json @@ -0,0 +1,10 @@ +{ + "uuid" : "56e1a32a-a33e-4b69-83dd-0bb8b1f80354", + "last_modified" : 0, + "name" : "ci_left_join_cube", + "owner" : null, + "descriptor" : "ci_left_join_cube", + "segments" : [ ], + "status" : "DISABLED", + "create_time" : null +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json new file mode 100644 index 0000000..48a051d --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -0,0 +1,286 @@ +{ + "uuid" : "3819ad72-3929-4dff-b59d-cd89a01238af", + "name" : "ci_inner_join_cube", + "model_name" : "ci_inner_join_model", + "description" : null, + "dimensions" : [ { + "name" : "ORDER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "ORDER_ID" + }, { + "name" : "CAL_DT", + "table" : "EDW.TEST_CAL_DT", + "column" : "{FK}", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "name" : "CATEGORY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "{FK}", + "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "META_CATEG_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL2_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL3_NAME", + "derived" : null + }, { + "name" : "LSTG_FORMAT_NAME", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "derived" : null + }, { + "name" : "SITE_ID", + "table" : "EDW.TEST_SITES", + "column" : "{FK}", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "name" : "SELLER_TYPE_CD", + "table" : "EDW.TEST_SELLER_TYPE_DIM", + "column" : "{FK}", + "derived" : [ "SELLER_TYPE_DESC" ] + }, { + "name" : "SELLER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "SELLER_ID", + "derived" : null + } ], + "measures" : [ { + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1" + }, + "returntype" : "bigint" + } + }, { + "name" : "ITEM_COUNT_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "ITEM_COUNT" + }, + "returntype" : "bigint" + } + }, { + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "SELLER_HLL", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "SELLER_ID" + }, + "returntype" : "hllc(10)" + } + }, { + "name" : "SELLER_FORMAT_HLL", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "LSTG_FORMAT_NAME", + "next_parameter" : { + "type" : "column", + "value" : "SELLER_ID" + } + }, + "returntype" : "hllc(10)" + } + }, { + "name" : "TOP_SELLER", + "function" : { + "expression" : "TOP_N", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : { + "type" : "column", + "value" : "SELLER_ID" + } + }, + "returntype" : "topn(100)", + "configuration": {"topn.encoding.SELLER_ID" : "int:4"} + } + }, { + "name" : "TEST_COUNT_DISTINCT_BITMAP", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "TEST_COUNT_DISTINCT_BITMAP" + }, + "returntype" : "bitmap" + } + }, { + "name" : "TEST_EXTENDED_COLUMN", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, { + "name" : "TRANS_ID_RAW", + "function" : { + "expression" : "RAW", + "parameter" : { + "type" : "column", + "value" : "TRANS_ID" + }, + "returntype" : "raw" + } + }, { + "name" : "PRICE_RAW", + "function" : { + "expression" : "RAW", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "raw" + } + }, { + "name" : "CAL_DT_RAW", + "function" : { + "expression" : "RAW", + "parameter" : { + "type" : "column", + "value" : "CAL_DT" + }, + "returntype" : "raw" + } + } ], + "dictionaries": [ { + "column": "TEST_COUNT_DISTINCT_BITMAP", + "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "SELLER_ID", + "encoding" : "int:4" + }, { + "column" : "ORDER_ID", + "encoding" : "int:4" + }, { + "column" : "CAL_DT", + "encoding" : "dict" + }, { + "column" : "LEAF_CATEG_ID", + "encoding" : "dict" + }, { + "column" : "META_CATEG_NAME", + "encoding" : "dict" + }, { + "column" : "CATEG_LVL2_NAME", + "encoding" : "dict" + }, { + "column" : "CATEG_LVL3_NAME", + "encoding" : "dict" + }, { + "column" : "LSTG_FORMAT_NAME", + "encoding" : "fixed_length:12" + }, { + "column" : "LSTG_SITE_ID", + "encoding" : "dict" + }, { + "column" : "SLR_SEGMENT_CD", + "encoding" : "dict" + } ] + }, + "signature" : null, + "last_modified" : 1448959801311, + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "f1", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ] + } ] + }, { + "name" : "f2", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ] + } ] + }, { + "name" : "f3", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ], + "select_rule" : { + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "mandatory_dims" : [ ], + "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + } + }, { + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "ORDER_ID" ], + "select_rule" : { + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID" ], + "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + } + } ], + "notify_list" : null, + "status_need_notify" : [ ], + "auto_merge_time_ranges" : null, + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "override_kylin_properties": { + "kylin.cube.algorithm": "LAYER" + }, + "partition_date_start": 0 +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json new file mode 100644 index 0000000..df370f7 --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json @@ -0,0 +1,286 @@ +{ + "uuid" : "629ab7a8-3929-4dff-b59d-2100aadccd1a", + "name" : "ci_left_join_cube", + "model_name" : "ci_left_join_model", + "description" : null, + "dimensions" : [ { + "name" : "ORDER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "ORDER_ID" + }, { + "name" : "CAL_DT", + "table" : "EDW.TEST_CAL_DT", + "column" : "{FK}", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "name" : "CATEGORY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "{FK}", + "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "META_CATEG_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL2_NAME", + "derived" : null + }, { + "name" : "CATEGORY_HIERARCHY", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL3_NAME", + "derived" : null + }, { + "name" : "LSTG_FORMAT_NAME", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "derived" : null + }, { + "name" : "SITE_ID", + "table" : "EDW.TEST_SITES", + "column" : "{FK}", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "name" : "SELLER_TYPE_CD", + "table" : "EDW.TEST_SELLER_TYPE_DIM", + "column" : "{FK}", + "derived" : [ "SELLER_TYPE_DESC" ] + }, { + "name" : "SELLER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "SELLER_ID", + "derived" : null + } ], + "measures" : [ { + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1" + }, + "returntype" : "bigint" + } + }, { + "name" : "ITEM_COUNT_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "ITEM_COUNT" + }, + "returntype" : "bigint" + } + }, { + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "SELLER_HLL", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "SELLER_ID" + }, + "returntype" : "hllc(10)" + } + }, { + "name" : "SELLER_FORMAT_HLL", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "LSTG_FORMAT_NAME", + "next_parameter" : { + "type" : "column", + "value" : "SELLER_ID" + } + }, + "returntype" : "hllc(10)" + } + }, { + "name" : "TOP_SELLER", + "function" : { + "expression" : "TOP_N", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : { + "type" : "column", + "value" : "SELLER_ID" + } + }, + "returntype" : "topn(100)", + "configuration": {"topn.encoding.SELLER_ID" : "int:4"} + } + }, { + "name" : "TEST_COUNT_DISTINCT_BITMAP", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "TEST_COUNT_DISTINCT_BITMAP" + }, + "returntype" : "bitmap" + } + }, { + "name" : "TEST_EXTENDED_COLUMN", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, { + "name" : "TRANS_ID_RAW", + "function" : { + "expression" : "RAW", + "parameter" : { + "type" : "column", + "value" : "TRANS_ID" + }, + "returntype" : "raw" + } + }, { + "name" : "PRICE_RAW", + "function" : { + "expression" : "RAW", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "raw" + } + }, { + "name" : "CAL_DT_RAW", + "function" : { + "expression" : "RAW", + "parameter" : { + "type" : "column", + "value" : "CAL_DT" + }, + "returntype" : "raw" + } + } ], + "dictionaries": [ { + "column": "TEST_COUNT_DISTINCT_BITMAP", + "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "SELLER_ID", + "encoding" : "int:4" + }, { + "column" : "ORDER_ID", + "encoding" : "int:4" + }, { + "column" : "CAL_DT", + "encoding" : "dict" + }, { + "column" : "LEAF_CATEG_ID", + "encoding" : "dict" + }, { + "column" : "META_CATEG_NAME", + "encoding" : "dict" + }, { + "column" : "CATEG_LVL2_NAME", + "encoding" : "dict" + }, { + "column" : "CATEG_LVL3_NAME", + "encoding" : "dict" + }, { + "column" : "LSTG_FORMAT_NAME", + "encoding" : "fixed_length:12" + }, { + "column" : "LSTG_SITE_ID", + "encoding" : "dict" + }, { + "column" : "SLR_SEGMENT_CD", + "encoding" : "dict" + } ] + }, + "signature" : null, + "last_modified" : 1448959801311, + "null_string" : null, + "hbase_mapping" : { + "column_family" : [ { + "name" : "f1", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ] + } ] + }, { + "name" : "f2", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ] + } ] + }, { + "name" : "f3", + "columns" : [ { + "qualifier" : "m", + "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ], + "select_rule" : { + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "mandatory_dims" : [ ], + "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + } + }, { + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "ORDER_ID" ], + "select_rule" : { + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID" ], + "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + } + } ], + "notify_list" : null, + "status_need_notify" : [ ], + "auto_merge_time_ranges" : null, + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "override_kylin_properties": { + "kylin.cube.algorithm": "INMEM" + }, + "partition_date_start": 0 +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json index 9c26be9..9aeaa2e 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json @@ -121,7 +121,7 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "USER_ID", + "value" : "ORDER_ID", "next_parameter" : null }, "returntype" : "bitmap" @@ -210,7 +210,7 @@ "dependent_measure_ref" : null } ], "dictionaries": [ { - "column": "USER_ID", + "column": "ORDER_ID", "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } ], "rowkey" : { http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json b/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json new file mode 100644 index 0000000..05d30c5 --- /dev/null +++ b/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json @@ -0,0 +1,14 @@ +{ + "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c", + + "name": "ci_inner_join_hybrid", + "realizations": [ + { + "type": "CUBE", + "realization": "ci_inner_join_cube" + } + ], + "cost": 100, + "last_modified": 1420016227424, + "create_time": null +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json deleted file mode 100644 index 443c0a3..0000000 --- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c", - - "name": "test_kylin_hybrid_inner_join", - "realizations": [ - { - "type": "CUBE", - "realization": "test_kylin_cube_without_slr_empty" - } - ], - "cost": 100, - "last_modified": 1420016227424, - "create_time": null -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json new file mode 100644 index 0000000..cf6b56b --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json @@ -0,0 +1,139 @@ +{ + "uuid": "72ab4ee2-2cdb-4b07-b39e-4c298563ae27", + "name": "ci_inner_join_model", + "fact_table": "DEFAULT.TEST_KYLIN_FACT", + "join_tables": [ + { + "table": "DEFAULT.TEST_ORDER", + "kind": "FACT", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_ORDER.ORDER_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.ORDER_ID" + ] + } + }, + { + "table": "EDW.TEST_CAL_DT", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_CAL_DT.CAL_DT" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.CAL_DT" + ] + } + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID", + "TEST_CATEGORY_GROUPINGS.SITE_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SITES", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_SITES.SITE_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "join": { + "type": "INNER", + "primary_key": [ + "TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" + ] + } + } + ], + "dimensions": [ + { + "table": "DEFAULT.TEST_KYLIN_FACT", + "columns": [ + "TRANS_ID", + "ORDER_ID", + "CAL_DT", + "LSTG_FORMAT_NAME", + "LSTG_SITE_ID", + "LEAF_CATEG_ID", + "SLR_SEGMENT_CD", + "SELLER_ID" + ] + }, + { + "table": "DEFAULT.TEST_ORDER", + "columns": [ + "ORDER_ID", + "BUYER_ID" + ] + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "columns": [ + "LEAF_CATEG_ID", + "SITE_ID", + "META_CATEG_NAME", + "CATEG_LVL2_NAME", + "CATEG_LVL3_NAME", + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER" + ] + }, + { + "table": "EDW.TEST_SITES", + "columns": [ + "SITE_ID", + "SITE_NAME", + "CRE_USER" + ] + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "columns": [ + "SELLER_TYPE_CD", + "SELLER_TYPE_DESC" + ] + }, + { + "table": "EDW.TEST_CAL_DT", + "columns": [ + "CAL_DT", + "WEEK_BEG_DT" + ] + } + ], + "metrics": [ + "TEST_KYLIN_FACT.PRICE", + "TEST_KYLIN_FACT.ITEM_COUNT" + ], + "last_modified": 1422435345352, + "filter_condition": null, + "partition_desc": { + "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.CAL_DT", + "partition_date_start": 0, + "partition_type": "APPEND" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json new file mode 100644 index 0000000..a3088ec --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json @@ -0,0 +1,141 @@ +{ + "uuid": "ed524ee2-2cdb-4b07-b39e-4c298563a863", + "name": "ci_left_join_model", + "fact_table": "DEFAULT.TEST_KYLIN_FACT", + "join_tables": [ + { + "table": "DEFAULT.TEST_ORDER", + "kind": "FACT", + "join": { + "type": "LEFT", + "primary_key": [ + "TEST_ORDER.ORDER_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.ORDER_ID" + ] + } + }, + { + "table": "EDW.TEST_CAL_DT", + "join": { + "type": "LEFT", + "primary_key": [ + "TEST_CAL_DT.CAL_DT" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.CAL_DT" + ] + } + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "join": { + "type": "LEFT", + "primary_key": [ + "TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID", + "TEST_CATEGORY_GROUPINGS.SITE_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SITES", + "join": { + "type": "LEFT", + "primary_key": [ + "TEST_SITES.SITE_ID" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "join": { + "type": "LEFT", + "primary_key": [ + "TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD" + ], + "foreign_key": [ + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" + ] + } + } + ], + "dimensions": [ + { + "table": "DEFAULT.TEST_KYLIN_FACT", + "columns": [ + "TRANS_ID", + "ORDER_ID", + "CAL_DT", + "LSTG_FORMAT_NAME", + "LSTG_SITE_ID", + "LEAF_CATEG_ID", + "SLR_SEGMENT_CD", + "SELLER_ID", + "TEST_EXTENDED_COLUMN", + "TEST_COUNT_DISTINCT_BITMAP" + ] + }, + { + "table": "DEFAULT.TEST_ORDER", + "columns": [ + "ORDER_ID", + "BUYER_ID" + ] + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "columns": [ + "LEAF_CATEG_ID", + "SITE_ID", + "META_CATEG_NAME", + "CATEG_LVL2_NAME", + "CATEG_LVL3_NAME", + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER" + ] + }, + { + "table": "EDW.TEST_SITES", + "columns": [ + "SITE_ID", + "SITE_NAME", + "CRE_USER" + ] + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "columns": [ + "SELLER_TYPE_CD", + "SELLER_TYPE_DESC" + ] + }, + { + "table": "EDW.TEST_CAL_DT", + "columns": [ + "CAL_DT", + "WEEK_BEG_DT" + ] + } + ], + "metrics": [ + "TEST_KYLIN_FACT.PRICE", + "TEST_KYLIN_FACT.ITEM_COUNT" + ], + "last_modified": 1422435345352, + "filter_condition": null, + "partition_desc": { + "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.CAL_DT", + "partition_date_start": 0, + "partition_type": "APPEND" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json index 16ce8ae..49758dd 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json @@ -63,7 +63,8 @@ "TRANS_ID", "CAL_DT", "LEAF_CATEG_ID", - "SELLER_ID" + "SELLER_ID", + "ORDER_ID" ] }, { @@ -105,9 +106,7 @@ ], "metrics": [ "PRICE", - "ITEM_COUNT", - "SELLER_ID", - "USER_ID" + "ITEM_COUNT" ], "last_modified": 1422435345352, "fact_table": "DEFAULT.TEST_KYLIN_FACT", http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json index 92b336f..e4cab89 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json @@ -63,7 +63,8 @@ "TRANS_ID", "CAL_DT", "LEAF_CATEG_ID", - "SELLER_ID" + "SELLER_ID", + "ORDER_ID" ] }, { @@ -105,9 +106,7 @@ ], "metrics": [ "PRICE", - "ITEM_COUNT", - "SELLER_ID", - "USER_ID" + "ITEM_COUNT" ], "last_modified": 1422435345352, "fact_table": "DEFAULT.TEST_KYLIN_FACT", http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 5eefce7..1ecc7cc 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -1,57 +1,49 @@ -{ - "uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b", - "name": "default", - "realizations": [ - { - "name": "test_kylin_cube_with_slr_empty", - "type": "CUBE", - "realization": "test_kylin_cube_with_slr_empty" - }, - { - "name": "test_kylin_cube_without_slr_empty", - "type": "CUBE", - "realization": "test_kylin_cube_without_slr_empty" - }, - { - "name": "test_kylin_cube_with_slr_left_join_empty", - "type": "CUBE", - "realization": "test_kylin_cube_with_slr_left_join_empty" - }, - { - "name": "test_kylin_cube_without_slr_left_join_empty", - "type": "CUBE", - "realization": "test_kylin_cube_without_slr_left_join_empty" - }, - { - "name": "test_streaming_table_cube", - "type": "CUBE", - "realization": "test_streaming_table_cube" - }, - { - "name": "test_kylin_hybrid_inner_join", - "type": "HYBRID", - "realization": "test_kylin_hybrid_inner_join" - }, +{ + "uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b", + "name": "default", + "realizations": [ + { + "name": "ci_left_join_cube", + "type": "CUBE", + "realization": "ci_left_join_cube" + }, + { + "name": "ci_inner_join_cube", + "type": "CUBE", + "realization": "ci_inner_join_cube" + }, + { + "name": "test_streaming_table_cube", + "type": "CUBE", + "realization": "test_streaming_table_cube" + }, + { + "name": "ci_inner_join_hybrid", + "type": "HYBRID", + "realization": "ci_inner_join_hybrid" + }, { "name": "ssb", "type": "CUBE", "realization": "ssb" - } - ], - "tables": [ - "DEFAULT.TEST_KYLIN_FACT", - "EDW.TEST_CAL_DT", - "DEFAULT.TEST_CATEGORY_GROUPINGS", - "EDW.TEST_SITES", - "EDW.TEST_SELLER_TYPE_DIM", - "EDW.TEST_SELLER_TYPE_DIM_TABLE", - "DEFAULT.STREAMING_TABLE" - ], - "models": [ - "ssb", - "test_kylin_inner_join_model_desc", - "test_kylin_left_join_model_desc", - "test_streaming_table_model_desc" + } + ], + "tables": [ + "DEFAULT.TEST_KYLIN_FACT", + "DEFAULT.TEST_ORDER", + "EDW.TEST_CAL_DT", + "DEFAULT.TEST_CATEGORY_GROUPINGS", + "EDW.TEST_SITES", + "EDW.TEST_SELLER_TYPE_DIM", + "DEFAULT.STREAMING_TABLE" + ], + "models": [ + "ssb", + "ci_inner_join_model", + "ci_left_join_model", + "test_kylin_inner_join_model_desc", + "test_kylin_left_join_model_desc", + "test_streaming_table_model_desc" ], "override_kylin_properties" :{ "kylin.storage.hbase.owner-tag": "ky...@kylin.apache.org" http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json index 5fb3edf..1aafaca 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json @@ -9,58 +9,57 @@ "data_gen" : "ID" }, { "id" : "2", + "name" : "ORDER_ID", + "datatype" : "bigint" + }, { + "id" : "3", "name" : "CAL_DT", "datatype" : "date", "data_gen" : "FK,order" }, { - "id" : "3", + "id" : "4", "name" : "LSTG_FORMAT_NAME", "datatype" : "string", "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others" }, { - "id" : "4", + "id" : "5", "name" : "LEAF_CATEG_ID", "datatype" : "bigint", "data_gen" : "FK,null,nullstr=0" }, { - "id" : "5", + "id" : "6", "name" : "LSTG_SITE_ID", "datatype" : "int" }, { - "id" : "6", + "id" : "7", "name" : "SLR_SEGMENT_CD", "datatype" : "smallint", "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD" }, { - "id" : "7", + "id" : "8", + "name" : "SELLER_ID", + "datatype" : "bigint", + "data_gen" : "RAND||10000000|10001000" + }, { + "id" : "9", "name" : "PRICE", "datatype" : "decimal", "data_gen" : "RAND|.##|-100|1000" }, { - "id" : "8", + "id" : "10", "name" : "ITEM_COUNT", "datatype" : "int", "data_gen" : "RAND" }, { - "id" : "9", - "name" : "SELLER_ID", - "datatype" : "bigint", - "data_gen" : "RAND||10000000|10001000" - }, { - "id" : "10", - "name" : "USER_ID", - "datatype" : "varchar(32)", - "data_gen" : "RAND,order" - }, { "id" : "11", - "name" : "BUYER_COUNTRY", + "name" : "TEST_EXTENDED_COLUMN", "datatype" : "string", - "data_gen" : "CN|DE|FR|JP|UK|US" + "data_gen" : "RAND" }, { "id" : "12", - "name" : "SELLER_COUNTRY", + "name" : "TEST_COUNT_DISTINCT_BITMAP", "datatype" : "string", - "data_gen" : "CN|DE|FR|JP|UK|US" + "data_gen" : "RAND" } ], "database" : "DEFAULT", "last_modified" : 0 http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json new file mode 100644 index 0000000..05647e6 --- /dev/null +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json @@ -0,0 +1,18 @@ +{ + "uuid" : "f216e39e-40d7-44c2-8fa2-41b3656398ae", + "name" : "TEST_ORDER", + "data_gen" : "0.5", + "columns" : [ { + "id" : "1", + "name" : "ORDER_ID", + "datatype" : "bigint", + "data_gen" : "ID" + }, { + "id" : "2", + "name" : "BUYER_ID", + "datatype" : "bigint", + "data_gen" : "RAND||10000500|10001500" + } ], + "database" : "DEFAULT", + "last_modified" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml index 9e8fc84..b05495f 100644 --- a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml +++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml @@ -78,19 +78,19 @@ <!--Additional config for in-mem cubing, giving mapper more memory --> <property> <name>mapreduce.map.memory.mb</name> - <value>512</value> + <value>768</value> <description></description> </property> <property> <name>mapreduce.map.java.opts</name> - <value>-Xmx384m</value> + <value>-Xmx700m</value> <description></description> </property> <property> <name>mapreduce.task.io.sort.mb</name> - <value>100</value> + <value>10</value> <description></description> </property> http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java index 8827dff..806bfab 100644 --- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java @@ -57,8 +57,8 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase { @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(ITDoggedCubeBuilderTest.class); - private static final int INPUT_ROWS = 10000; - private static final int SPLIT_ROWS = 5000; + private static final int INPUT_ROWS = 4000; + private static final int SPLIT_ROWS = 2000; private static final int THREADS = 4; private static CubeInstance cube; http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java index dea9f2d..f9b6ef2 100644 --- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java @@ -85,7 +85,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase { @Test public void testKylinCube() throws Exception { testBuild("test_kylin_cube_without_slr_left_join_empty", // - LOCALMETA_TEST_DATA + "/data/flatten_data_for_without_slr_left_join.csv", 70000, 4); + LOCALMETA_TEST_DATA + "/data/flatten_data_for_without_slr_left_join.csv", 7000, 4); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java index bfbeb70..31a5cee 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java @@ -76,6 +76,8 @@ public class BuildCubeWithEngine { private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithEngine.class); public static void main(String[] args) throws Exception { + long start = System.currentTimeMillis(); + int exitCode = 0; try { beforeClass(); @@ -86,11 +88,15 @@ public class BuildCubeWithEngine { logger.info("Build is done"); afterClass(); logger.info("Going to exit"); - System.exit(0); } catch (Throwable e) { logger.error("error", e); - System.exit(1); + exitCode = 1; } + + long millis = System.currentTimeMillis() - start; + System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName()); + + System.exit(exitCode); } public static void beforeClass() throws Exception { @@ -161,7 +167,7 @@ public class BuildCubeWithEngine { } public void build() throws Exception { - DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty"); + DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model"); KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(1.0f); testInner(); testLeft(); @@ -184,12 +190,12 @@ public class BuildCubeWithEngine { } private void testInner() throws Exception { - String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" }; + String[] testCase = new String[] { "testInnerJoinCube" }; runTestAndAssertSucceed(testCase); } private void testLeft() throws Exception { - String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" }; + String[] testCase = new String[] { "testLeftJoinCube" }; runTestAndAssertSucceed(testCase); } @@ -243,47 +249,24 @@ public class BuildCubeWithEngine { @SuppressWarnings("unused") // called by reflection - private boolean testInnerJoinCubeWithSlr() throws Exception { - final String cubeName = "test_kylin_cube_with_slr_empty"; - clearSegment(cubeName); - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = 0; - long date2 = f.parse("2015-01-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); - - if (fastBuildMode) { - return buildSegment(cubeName, date1, date3); - } else { - if (buildSegment(cubeName, date1, date2) == true) { - return buildSegment(cubeName, date2, date3);//empty segment - } - } - return false; - } - - @SuppressWarnings("unused") - // called by reflection - private boolean testLeftJoinCubeWithSlr() throws Exception { - String cubeName = "test_kylin_cube_with_slr_left_join_empty"; + private boolean testLeftJoinCube() throws Exception { + String cubeName = "ci_left_join_cube"; clearSegment(cubeName); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); + long date1 = 0; long date2 = f.parse("2013-01-01").getTime(); long date3 = f.parse("2013-07-01").getTime(); long date4 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); if (fastBuildMode) { return buildSegment(cubeName, date1, date4); } else { if (buildSegment(cubeName, date1, date2) == true) { if (buildSegment(cubeName, date2, date3) == true) { if (buildSegment(cubeName, date3, date4) == true) { - return mergeSegment(cubeName, date1, date3);//don't merge all segments + return mergeSegment(cubeName, date1, date3); // don't merge all segments } } } @@ -293,61 +276,25 @@ public class BuildCubeWithEngine { @SuppressWarnings("unused") // called by reflection - private boolean testInnerJoinCubeWithoutSlr() throws Exception { + private boolean testInnerJoinCube() throws Exception { - final String cubeName = "test_kylin_cube_without_slr_empty"; + String cubeName = "ci_inner_join_cube"; clearSegment(cubeName); + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); f.setTimeZone(TimeZone.getTimeZone("GMT")); long date1 = 0; - long date2 = f.parse("2013-01-01").getTime(); - long date3 = f.parse("2013-07-01").getTime(); - long date4 = f.parse("2022-01-01").getTime(); - List<String> result = Lists.newArrayList(); + long date2 = f.parse("2022-07-01").getTime(); + long date3 = f.parse("2023-01-01").getTime(); if (fastBuildMode) { - return buildSegment(cubeName, date1, date4); - } else { - if (buildSegment(cubeName, date1, date2) == true) { - if (buildSegment(cubeName, date2, date3) == true) { - if (buildSegment(cubeName, date3, date4) == true) { - return mergeSegment(cubeName, date1, date3);//don't merge all segments - } - } - } - } - return false; - - } - - @SuppressWarnings("unused") - // called by reflection - private boolean testLeftJoinCubeWithoutSlr() throws Exception { - SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); - f.setTimeZone(TimeZone.getTimeZone("GMT")); - List<String> result = Lists.newArrayList(); - final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; - clearSegment(cubeName); - - long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart(); - long date2 = f.parse("2012-06-01").getTime(); - long date3 = f.parse("2022-01-01").getTime(); - long date4 = f.parse("2023-01-01").getTime(); - - if (fastBuildMode) { - return buildSegment(cubeName, date1, date4); + return buildSegment(cubeName, date1, date3); } else { - if (buildSegment(cubeName, date1, date2) == true) { - if (buildSegment(cubeName, date2, date3) == true) { - if (buildSegment(cubeName, date3, date4) == true) { //empty segment - return mergeSegment(cubeName, date1, date3);//don't merge all segments - } - } + if (buildSegment(cubeName, date1, date2) == true) { // all-in-one build + return buildSegment(cubeName, date2, date3); // empty segment } } - return false; - } private void clearSegment(String cubeName) throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index 5f1e3a9..23eadaf 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -63,17 +63,14 @@ public class ITCombinationTest extends ITKylinQueryTest { @Parameterized.Parameters public static Collection<Object[]> configs() { return Arrays.asList(new Object[][] { // - { "inner", "on", "v2", true }, // - { "left", "on", "v2", true }, // - //{ "inner", "on", "v2", false }, // exclude view to simply model/cube selection - //{ "left", "on", "v1", false }, // exclude view to simply model/cube selection - //{ "left", "on", "v2", false }, // exclude view to simply model/cube selection + { "inner", "on", "v2" }, // + { "left", "on", "v2" }, // }); } - public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception { + public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine) throws Exception { - printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine + ", excludeViewCubes: " + excludeViewCubes); + printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine); ITKylinQueryTest.clean(); @@ -81,9 +78,5 @@ public class ITCombinationTest extends ITKylinQueryTest { ITKylinQueryTest.setupAll(); RemoveBlackoutRealizationsRule.blackList.clear(); - if (excludeViewCubes) { - RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]"); - RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]"); - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 3a7b685..227fa0a 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -62,12 +62,9 @@ public class ITKylinQueryTest extends KylinTestBase { priorities.put(RealizationType.INVERTED_INDEX, 0); Candidate.setPriorities(priorities); - joinType = "inner"; + joinType = "left"; setupAll(); - - RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]"); - RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]"); } @AfterClass @@ -152,11 +149,11 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { - String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query02.sql"; + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_limit/query01.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { - runSQL(sqlFile, true, true); + //runSQL(sqlFile, true, true); runSQL(sqlFile, true, false); } } @@ -324,18 +321,10 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testLimitEnabled() throws Exception { - try { - //other cubes have strange aggregation groups - RemoveBlackoutRealizationsRule.whiteList.add("CUBE[name=test_kylin_cube_with_slr_empty]"); - - List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql"); - for (File sqlFile : sqlFiles) { - runSQL(sqlFile, false, false); - assertTrue(checkFinalPushDownLimit()); - } - - } finally { - RemoveBlackoutRealizationsRule.whiteList.remove("CUBE[name=test_kylin_cube_with_slr_empty]"); + List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql"); + for (File sqlFile : sqlFiles) { + runSQL(sqlFile, false, false); + assertTrue(checkFinalPushDownLimit()); } } @@ -381,7 +370,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSelectStarColumnCount() throws Exception { - execAndCompColumnCount("select * from test_kylin_fact limit 10", 9); - execAndCompColumnCount("select * from test_kylin_fact", 9); + execAndCompColumnCount("select * from test_kylin_fact limit 10", 12); + execAndCompColumnCount("select * from test_kylin_fact", 12); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_derived/query12.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql b/kylin-it/src/test/resources/query/sql_derived/query12.sql index 959a59c..f1f331c 100755 --- a/kylin-it/src/test/resources/query/sql_derived/query12.sql +++ b/kylin-it/src/test/resources/query/sql_derived/query12.sql @@ -18,7 +18,7 @@ select UPD_USER,count(1) as CNT from TEST_KYLIN_FACT as TEST_KYLIN_FACT -JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +inner join TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID where UPD_USER not in ('USER_Y') group by UPD_USER \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql index 0c163a6..762ea38 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql @@ -19,6 +19,6 @@ select lstg_format_name, cal_dt, sum(price) as GMV, count(1) as TRANS_CNT, - count(distinct user_id) as user_count + count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count from test_kylin_fact group by lstg_format_name, cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql index 62142a2..b05f022 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql @@ -19,7 +19,7 @@ select lstg_format_name, sum(price) as GMV, count(1) as TRANS_CNT, - count(distinct user_id) as user_count + count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count from test_kylin_fact where lstg_format_name='FP-GTC' group by lstg_format_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql index 88ce532..11418d1 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql @@ -19,7 +19,7 @@ select lstg_format_name, sum(price) as GMV, count(1) as TRANS_CNT, - count(distinct user_id) as user_count + count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count from test_kylin_fact where lstg_format_name='FP-GTC' group by lstg_format_name http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql index 7f739e7..2be8963 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql @@ -18,7 +18,7 @@ select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV , count(1) as TRANS_CNT - , count(distinct user_id) as user_count + , count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count , count(distinct site_name) as site_count from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql index 13de5a7..7306824 100644 --- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql +++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql @@ -18,7 +18,7 @@ select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV , count(1) as TRANS_CNT - , count(distinct user_id) as user_count + , count(distinct TEST_COUNT_DISTINCT_BITMAP) as user_count , count(distinct site_name) as site_count from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql b/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql index 15e274a..62d34da 100644 --- a/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql +++ b/kylin-it/src/test/resources/query/sql_intersect_count/query00.sql @@ -18,13 +18,13 @@ select week_beg_dt as week, -intersect_count( seller_id, lstg_format_name, array['FP-GTC']) as a, -intersect_count( seller_id, lstg_format_name, array['Auction']) as b, -intersect_count( seller_id, lstg_format_name, array['Others']) as c, -intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Auction']) as ab, -intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Others']) as ac, -intersect_count( seller_id, lstg_format_name, array['FP-GTC', 'Auction', 'Others']) as abc, -count(distinct seller_id) as sellers, +intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC']) as a, +intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['Auction']) as b, +intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['Others']) as c, +intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC', 'Auction']) as ab, +intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC', 'Others']) as ac, +intersect_count( TEST_COUNT_DISTINCT_BITMAP, lstg_format_name, array['FP-GTC', 'Auction', 'Others']) as abc, +count(distinct TEST_COUNT_DISTINCT_BITMAP) as sellers, count(*) as cnt from test_kylin_fact left join edw.test_cal_dt on test_kylin_fact.cal_dt = edw.test_cal_dt.CAL_DT where week_beg_dt in (DATE '2013-12-22', DATE '2012-06-23') http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_limit/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_limit/query01.sql b/kylin-it/src/test/resources/query/sql_limit/query01.sql index fca8175..0752165 100644 --- a/kylin-it/src/test/resources/query/sql_limit/query01.sql +++ b/kylin-it/src/test/resources/query/sql_limit/query01.sql @@ -16,6 +16,10 @@ -- limitations under the License. -- -select * from test_kylin_fact - where lstg_format_name='FP-GTC' - limit 20 +select + * +from test_kylin_fact + inner join TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS + ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +where lstg_format_name='FP-GTC' +limit 20 http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query04.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_raw/query04.sql b/kylin-it/src/test/resources/query/sql_raw/query04.sql index 2dc7d25..22f6ac2 100644 --- a/kylin-it/src/test/resources/query/sql_raw/query04.sql +++ b/kylin-it/src/test/resources/query/sql_raw/query04.sql @@ -16,4 +16,4 @@ -- limitations under the License. -- -select CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN' +select CAL_DT,LSTG_FORMAT_NAME,PRICE,TRANS_ID from test_kylin_fact where LSTG_FORMAT_NAME = 'ABIN' http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query06.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_raw/query06.sql b/kylin-it/src/test/resources/query/sql_raw/query06.sql index 0701c1f..7080ab5 100644 --- a/kylin-it/src/test/resources/query/sql_raw/query06.sql +++ b/kylin-it/src/test/resources/query/sql_raw/query06.sql @@ -16,4 +16,4 @@ -- limitations under the License. -- -select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,CAL_DT,LEAF_CATEG_ID,PRICE from test_kylin_fact \ No newline at end of file +select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,CAL_DT,LEAF_CATEG_ID,PRICE,TRANS_ID from test_kylin_fact \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query23.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_raw/query23.sql b/kylin-it/src/test/resources/query/sql_raw/query23.sql index 89d61db..5be4ad0 100644 --- a/kylin-it/src/test/resources/query/sql_raw/query23.sql +++ b/kylin-it/src/test/resources/query/sql_raw/query23.sql @@ -16,7 +16,7 @@ -- limitations under the License. -- -select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt +select test_kylin_fact.CAL_DT,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt inner JOIN test_category_groupings ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fbffd5/kylin-it/src/test/resources/query/sql_raw/query24.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_raw/query24.sql b/kylin-it/src/test/resources/query/sql_raw/query24.sql deleted file mode 100644 index 46c7329..0000000 --- a/kylin-it/src/test/resources/query/sql_raw/query24.sql +++ /dev/null @@ -1,24 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - inner JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - inner JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
