KYLIN-1875 Add JoinsTree, refactor ModelChooser
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bda8787d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bda8787d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bda8787d Branch: refs/heads/master-hbase1.x Commit: bda8787d1a5c9bbe7d94798091763e1004a1eb61 Parents: a1a2a4a Author: Yang Li <liy...@apache.org> Authored: Mon Nov 28 07:49:11 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Tue Nov 29 07:25:29 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/dict/DictionaryManager.java | 6 +- .../kylin/metadata/model/DataModelDesc.java | 30 +- .../apache/kylin/metadata/model/JoinDesc.java | 11 + .../apache/kylin/metadata/model/JoinsTree.java | 130 ++++++ .../apache/kylin/metadata/model/TblColRef.java | 4 + .../kylin/metadata/MetadataManagerTest.java | 11 +- .../kylin/metadata/model/JoinsTreeTest.java | 74 ++++ .../test_kylin_snowflake_sales_cube.json | 268 ++++++++++++ .../test_kylin_snowflake_model_desc.json | 161 -------- .../test_kylin_snowflake_sales_model.json | 87 ++++ .../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 ++ .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 +++++++++++++++++++ .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 +++++++ .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 ++ .../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 +++ .../apache/kylin/query/relnode/OLAPContext.java | 2 + .../kylin/query/relnode/OLAPTableScan.java | 7 +- .../kylin/query/routing/ModelChooser.java | 55 +-- 18 files changed, 1293 insertions(+), 221 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index 820299c..6178234 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -339,7 +339,7 @@ public class DictionaryManager { // find a lookup table that the col joins as FK for (TableRef lookup : model.getLookupTables()) { - JoinDesc lookupJoin = model.getPKSideJoinMap().get(lookup); + JoinDesc lookupJoin = model.getJoinByPKSide(lookup); int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(), col); if (find < 0) continue; @@ -357,8 +357,8 @@ public class DictionaryManager { if (join.isInnerJoin() == false) return false; - TableRef table = join.getForeignKeyColumns()[0].getTableRef(); - join = model.getPKSideJoinMap().get(table); + TableRef table = join.getFKSide(); + join = model.getJoinByPKSide(table); } return true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index ff92def..d917571 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -97,8 +97,7 @@ public class DataModelDesc extends RootPersistentEntity { private Set<TableRef> allTableRefs = Sets.newLinkedHashSet(); private Map<String, TableRef> aliasMap = Maps.newHashMap(); // alias => TableRef, a table has exactly one alias private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // name => TableRef, a table maybe referenced by multiple names - private Map<TableRef, JoinDesc> pkSideJoinMap = Maps.newHashMap(); // table (PK side) => JoinTable - private Map<String, List<JoinDesc>> fkSideJoinMap = Maps.newHashMap(); // table (FK side) => JoinDesc + private JoinsTree joinsTree; /** * Error messages during resolving json metadata @@ -151,12 +150,12 @@ public class DataModelDesc extends RootPersistentEntity { return joinTables; } - public Map<TableRef, JoinDesc> getPKSideJoinMap() { - return pkSideJoinMap; + public JoinDesc getJoinByPKSide(TableRef table) { + return joinsTree.getJoinByPKSide(table); } - - public Map<String, List<JoinDesc>> getFKSideJoinMap() { - return fkSideJoinMap; + + public JoinsTree getJoinsTree() { + return joinsTree; } @Deprecated @@ -282,6 +281,7 @@ public class DataModelDesc extends RootPersistentEntity { initJoinTablesForUpgrade(); initTableAlias(tables); initJoinColumns(); + initJoinsTree(); ModelDimensionDesc.capicalizeStrings(dimensions); initPartitionDesc(); } @@ -360,8 +360,6 @@ public class DataModelDesc extends RootPersistentEntity { } private void initJoinColumns() { - pkSideJoinMap.clear(); - fkSideJoinMap.clear(); for (JoinTableDesc joinTable : joinTables) { TableRef dimTable = joinTable.getTableRef(); @@ -413,15 +411,15 @@ public class DataModelDesc extends RootPersistentEntity { logger.warn("PK " + dimTable + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with FK " + fkTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype()); } } + } + } - // pk/fk side join maps - pkSideJoinMap.put(dimTable, join); - List<JoinDesc> list = fkSideJoinMap.get(fkTable.getTableIdentity()); - if (list == null) { - fkSideJoinMap.put(fkTable.getTableIdentity(), list = Lists.newArrayListWithCapacity(4)); - } - list.add(join); + private void initJoinsTree() { + List<JoinDesc> joins = new ArrayList<>(); + for (JoinTableDesc joinTable : joinTables) { + joins.add(joinTable.getJoin()); } + joinsTree = new JoinsTree(rootFactTableRef, joins); } /** http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java index d3c0745..6489244 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java @@ -110,6 +110,14 @@ public class JoinDesc { Preconditions.checkState(tableRef == cols[i].getTableRef()); } + public TableRef getPKSide() { + return primaryKeyColumns[0].getTableRef(); + } + + public TableRef getFKSide() { + return foreignKeyColumns[0].getTableRef(); + } + public void sortByFK() { Preconditions.checkState(primaryKey.length == foreignKey.length && primaryKey.length == primaryKeyColumns.length && foreignKey.length == foreignKeyColumns.length); boolean cont = true; @@ -174,6 +182,9 @@ public class JoinDesc { // equals() without alias public boolean matches(JoinDesc other) { + if (other == null) + return false; + if (!this.type.equalsIgnoreCase(other.getType())) return false; http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java new file mode 100644 index 0000000..a0b267d --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.metadata.model; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.google.common.base.Preconditions; + +public class JoinsTree { + + final Map<String, Chain> tableChains = new LinkedHashMap<>(); + + public JoinsTree(TableRef rootTable, List<JoinDesc> joins) { + for (JoinDesc join : joins) { + for (TblColRef col : join.getForeignKeyColumns()) + Preconditions.checkState(col.isQualified()); + for (TblColRef col : join.getPrimaryKeyColumns()) + Preconditions.checkState(col.isQualified()); + } + + tableChains.put(rootTable.getAlias(), new Chain(rootTable, null, null)); + + for (JoinDesc join : joins) { + TableRef pkSide = join.getPKSide(); + Chain fkSide = tableChains.get(join.getFKSide().getAlias()); + tableChains.put(pkSide.getAlias(), new Chain(pkSide, join, fkSide)); + } + } + + public Map<String, String> matches(JoinsTree another) { + return matches(another, Collections.<String, String> emptyMap()); + } + + public Map<String, String> matches(JoinsTree another, Map<String, String> constraints) { + Map<String, String> matchUp = new HashMap<>(); + + for (Chain chain : tableChains.values()) { + if (matchInTree(chain, another, constraints, matchUp) == false) + return null; + } + + return matchUp; + } + + private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, Map<String, String> matchUp) { + String thisAlias = chain.table.getAlias(); + if (matchUp.containsKey(thisAlias)) + return true; + + String constraint = constraints.get(thisAlias); + if (constraint != null) { + return matchChain(chain, another.tableChains.get(constraint), matchUp); + } + + for (Chain anotherChain : another.tableChains.values()) { + if (matchChain(chain, anotherChain, matchUp)) { + return true; + } + } + return false; + } + + private boolean matchChain(Chain chain, Chain anotherChain, Map<String, String> matchUp) { + String thisAlias = chain.table.getAlias(); + String anotherAlias = anotherChain.table.getAlias(); + + String curMatch = matchUp.get(thisAlias); + if (curMatch != null) + return curMatch.equals(anotherAlias); + if (curMatch == null && matchUp.values().contains(anotherAlias)) + return false; + + boolean matches = false; + if (chain.join == null) { + matches = anotherChain.join == null && chain.table.getTableDesc().equals(anotherChain.table.getTableDesc()); + } else { + matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp); + } + + if (matches) { + matchUp.put(thisAlias, anotherAlias); + } + return matches; + } + + public JoinDesc getJoinByPKSide(TableRef table) { + Chain chain = tableChains.get(table.getAlias()); + if (chain == null) + return null; + else + return chain.join; + } + + static class Chain { + TableRef table; // pk side + JoinDesc join; + Chain fkSide; + + public Chain(TableRef table, JoinDesc join, Chain fkSide) { + this.table = table; + this.join = join; + this.fkSide = fkSide; + if (join != null) { + Preconditions.checkArgument(table == join.getPKSide()); + Preconditions.checkArgument(fkSide.table == join.getFKSide()); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index 2cfbafc..bf8d36b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -123,6 +123,10 @@ public class TblColRef implements Serializable { return table; } + public boolean isQualified() { + return table != null; + } + public String getTableAlias() { return table != null ? table.getAlias() : "UNKNOWN_ALIAS"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java index fcae8d4..204b4e0 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java @@ -76,17 +76,20 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { @Test public void testSnowflakeDataModel() throws Exception { - DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_model_desc"); + DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model"); Assert.assertTrue(model.getDimensions().size() > 0); try { - model.findTable("TEST_KYLIN_COUNTRY"); + model.findTable("KYLIN_COUNTRY"); Assert.fail(); } catch (IllegalArgumentException ex) { // excepted } - Assert.assertNotNull(model.findColumn("BUYER_COUNTRY")); - Assert.assertNotNull(model.findColumn("SELLER_COUNTRY")); + + Assert.assertNotNull(model.findTable("BUYER_COUNTRY")); + Assert.assertNotNull(model.findTable("SELLER_COUNTRY")); + Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME")); + Assert.assertNotNull(model.findColumn("BUYER_ID")); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java new file mode 100644 index 0000000..f52ef45 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.metadata.model; + +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.JoinsTree.Chain; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + */ +public class JoinsTreeTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void testBasics() { + MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model"); + JoinsTree joinsTree = model.getJoinsTree(); + + Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY"); + assertTrue(chain.table == model.findTable("BUYER_COUNTRY")); + assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT")); + assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES")); + assertTrue(chain.fkSide.fkSide.join == null); + assertTrue(chain.fkSide.fkSide.fkSide == null); + } + + @Test + public void testMatch() { + MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model"); + JoinsTree joinsTree = model.getJoinsTree(); + + Map<String, String> matchUp = joinsTree.matches(joinsTree); + for (Entry<String, String> e : matchUp.entrySet()) { + assertTrue(e.getKey().equals(e.getValue())); + } + assertTrue(model.getAllTables().size() == matchUp.size()); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json new file mode 100644 index 0000000..ee0d68c --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json @@ -0,0 +1,268 @@ +{ + "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf", + "last_modified" : 1451468470824, + "name" : "test_kylin_snowflake_sales_cube", + "model_name" : "test_kylin_snowflake_sales_model", + "description" : null, + "null_string" : null, + "dimensions" : [ { + "name" : "TRANS_ID", + "table" : "SNOWTEST.KYLIN_SALES", + "column" : "TRANS_ID", + "derived" : null + }, { + "name" : "CAL_DT", + "table" : "SNOWTEST.KYLIN_CAL_DT", + "column" : "{FK}", + "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ] + }, { + "name" : "CATEGORY", + "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", + "column" : "{FK}", + "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ] + }, { + "name" : "META_CATEG_NAME", + "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", + "column" : "META_CATEG_NAME", + "derived" : null + }, { + "name" : "CATEG_LVL2_NAME", + "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL2_NAME", + "derived" : null + }, { + "name" : "CATEG_LVL3_NAME", + "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", + "column" : "CATEG_LVL3_NAME", + "derived" : null + }, { + "name" : "LSTG_FORMAT_NAME", + "table" : "SNOWTEST.KYLIN_SALES", + "column" : "LSTG_FORMAT_NAME", + "derived" : null + }, { + "name" : "SELLER_ID", + "table" : "SNOWTEST.KYLIN_SALES", + "column" : "SELLER_ID", + "derived" : null + }, { + "name" : "BUYER_ID", + "table" : "SNOWTEST.KYLIN_SALES", + "column" : "BUYER_ID", + "derived" : null + }, { + "name" : "ACCOUNT_BUYER_LEVEL", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_BUYER_LEVEL", + "derived" : null + }, { + "name" : "ACCOUNT_SELLER_LEVEL", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_SELLER_LEVEL", + "derived" : null + }, { + "name" : "BUYER_COUNTRY", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_COUNTRY", + "derived" : null + }, { + "name" : "SELLER_COUNTRY", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_COUNTRY", + "derived" : null + }, { + "name" : "BUYER_COUNTRY_NAME", + "table" : "BUYER_COUNTRY", + "column" : "NAME", + "derived" : null + }, { + "name" : "SELLER_COUNTRY_NAME", + "table" : "SELLER_COUNTRY", + "column" : "NAME", + "derived" : null + }, { + "name" : "OPS_USER_ID", + "table" : "SNOWTEST.KYLIN_SALES", + "column" : "OPS_USER_ID", + "derived" : null + }, { + "name" : "OPS_REGION", + "table" : "SNOWTEST.KYLIN_SALES", + "column" : "OPS_REGION", + "derived" : null + } ], + "measures" : [ { + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : null + }, + "returntype" : "decimal(19,4)" + } + }, { + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + } + }, { + "name" : "SELLER_CNT_HLL", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "SELLER_ID", + "next_parameter" : null + }, + "returntype" : "hllc(10)" + } + }, { + "name" : "TOP_SELLER", + "function" : { + "expression" : "TOP_N", + "parameter" : { + "type" : "column", + "value" : "PRICE", + "next_parameter" : { + "type" : "column", + "value" : "SELLER_ID", + "next_parameter" : null + } + }, + "returntype" : "topn(100)" + } + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "BUYER_ID", + "encoding" : "integer:4" + }, { + "column" : "SELLER_ID", + "encoding" : "integer:4" + }, { + "column" : "TRANS_ID", + "encoding" : "integer:4" + }, { + "column" : "PART_DT", + "encoding" : "date" + }, { + "column" : "LEAF_CATEG_ID", + "encoding" : "dict" + }, { + "column" : "META_CATEG_NAME", + "encoding" : "dict" + }, { + "column" : "CATEG_LVL2_NAME", + "encoding" : "dict" + }, { + "column" : "CATEG_LVL3_NAME", + "encoding" : "dict" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding" : "dict" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding" : "dict" + }, { + "column" : "BUYER_COUNTRY.NAME", + "encoding" : "dict" + }, { + "column" : "SELLER_COUNTRY.NAME", + "encoding" : "dict" + }, { + "column" : "LSTG_FORMAT_NAME", + "encoding" : "dict" + }, { + "column" : "LSTG_SITE_ID", + "encoding" : "dict" + }, { + "column" : "OPS_USER_ID", + "encoding" : "dict" + }, { + "column" : "OPS_REGION", + "encoding" : "dict" + } ] + }, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ] + } ] + }, { + "name" : "F2", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ], + "select_rule" : { + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "mandatory_dims" : [ "PART_DT" ], + "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ], + [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ] + } + }, { + "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID", + "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ], + "select_rule" : { + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID", + "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ], + "joint_dims" : [ ] + } + } ], + "notify_list" : null, + "status_need_notify" : [ ], + "partition_date_start" : 1325376000000, + "auto_merge_time_ranges" : null, + "retention_range" : 0, + "engine_type" : 2, + "storage_type" : 2, + "override_kylin_properties" : { + "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true" + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json deleted file mode 100644 index b06a217..0000000 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "uuid": "ac0f4ee2-1dcb-4b07-a38e-4c298563e0e3", - "name": "test_kylin_snowflake_model_desc", - "lookups": [ - { - "table": "EDW.TEST_CAL_DT", - "join": { - "type": "left", - "primary_key": [ - "CAL_DT" - ], - "foreign_key": [ - "CAL_DT" - ] - } - }, - { - "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", - "join": { - "type": "left", - "primary_key": [ - "LEAF_CATEG_ID", - "SITE_ID" - ], - "foreign_key": [ - "LEAF_CATEG_ID", - "LSTG_SITE_ID" - ] - } - }, - { - "table": "EDW.TEST_SITES", - "join": { - "type": "left", - "primary_key": [ - "SITE_ID" - ], - "foreign_key": [ - "LSTG_SITE_ID" - ] - } - }, - { - "table": "EDW.TEST_SELLER_TYPE_DIM", - "join": { - "type": "left", - "primary_key": [ - "SELLER_TYPE_CD" - ], - "foreign_key": [ - "SLR_SEGMENT_CD" - ] - } - }, - { - "table": "DEFAULT.TEST_KYLIN_COUNTRY", - "alias": "BUYER_COUNTRY", - "join": { - "type": "left", - "primary_key": [ - "COUNTRY" - ], - "foreign_key": [ - "BUYER_COUNTRY" - ] - } - }, - { - "table": "DEFAULT.TEST_KYLIN_COUNTRY", - "alias": "SELLER_COUNTRY", - "join": { - "type": "left", - "primary_key": [ - "COUNTRY" - ], - "foreign_key": [ - "SELLER_COUNTRY" - ] - } - } - ], - "dimensions": [ - { - "table": "default.test_kylin_fact", - "columns": [ - "lstg_format_name", - "LSTG_SITE_ID", - "SLR_SEGMENT_CD", - "TRANS_ID", - "CAL_DT", - "LEAF_CATEG_ID", - "SELLER_ID", - "BUYER_COUNTRY", - "SELLER_COUNTRY" - ] - }, - { - "table": "default.test_category_groupings", - "columns": [ - "leaf_categ_id", - "site_id", - "USER_DEFINED_FIELD1", - "USER_DEFINED_FIELD3", - "UPD_DATE", - "UPD_USER", - "meta_categ_name", - "categ_lvl2_name", - "categ_lvl3_name" - ] - }, - { - "table": "edw.test_sites", - "columns": [ - "site_id", - "site_name", - "cre_user" - ] - }, - { - "table": "edw.test_seller_type_dim", - "columns": [ - "seller_type_cd", - "seller_type_desc" - ] - }, - { - "table": "edw.test_cal_dt", - "columns": [ - "cal_dt", - "week_beg_dt" - ] - }, - { - "table": "BUYER_COUNTRY", - "columns": [ - "country", - "name" - ] - }, - { - "table": "SELLER_COUNTRY", - "columns": [ - "country", - "name" - ] - } - ], - "metrics": [ - "PRICE", - "ITEM_COUNT", - "SELLER_ID" - ], - "last_modified": 1422435345352, - "fact_table": "DEFAULT.TEST_KYLIN_FACT", - "filter_condition": null, - "partition_desc": { - "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt", - "partition_date_start": 0, - "partition_type": "APPEND" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json new file mode 100644 index 0000000..84c6979 --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json @@ -0,0 +1,87 @@ +{ + "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f", + "name" : "test_kylin_snowflake_sales_model", + "lookups" : [ { + "table" : "SNOWTEST.KYLIN_CAL_DT", + "join" : { + "type" : "inner", + "primary_key" : [ "CAL_DT" ], + "foreign_key" : [ "PART_DT" ] + } + }, { + "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", + "join" : { + "type" : "inner", + "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], + "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] + } + }, { + "table" : "SNOWTEST.KYLIN_ACCOUNT", + "alias" : "BUYER_ACCOUNT", + "join" : { + "type" : "inner", + "primary_key" : [ "ACCOUNT_ID" ], + "foreign_key" : [ "BUYER_ID" ] + } + }, { + "table" : "SNOWTEST.KYLIN_ACCOUNT", + "alias" : "SELLER_ACCOUNT", + "join" : { + "type" : "inner", + "primary_key" : [ "ACCOUNT_ID" ], + "foreign_key" : [ "SELLER_ID" ] + } + }, { + "table" : "SNOWTEST.KYLIN_COUNTRY", + "alias" : "BUYER_COUNTRY", + "join" : { + "type" : "inner", + "primary_key" : [ "COUNTRY" ], + "foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ] + } + }, { + "table" : "SNOWTEST.KYLIN_COUNTRY", + "alias" : "SELLER_COUNTRY", + "join" : { + "type" : "inner", + "primary_key" : [ "COUNTRY" ], + "foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ] + } + }], + "dimensions" : [ { + "table" : "SNOWTEST.KYLIN_SALES", + "columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ] + }, { + "table" : "SNOWTEST.KYLIN_CAL_DT", + "columns" : ["CAL_DT", "WEEK_BEG_DT"] + }, { + "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", + "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ] + }, { + "table" : "BUYER_ACCOUNT", + "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ] + }, { + "table" : "SELLER_ACCOUNT", + "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ] + }, { + "table" : "BUYER_COUNTRY", + "columns" : [ "COUNTRY", "NAME" ] + }, { + "table" : "SELLER_COUNTRY", + "columns" : [ "COUNTRY", "NAME" ] + } ], + "metrics": [ + "PRICE", + "ITEM_COUNT", + "SELLER_ID", + "LSTG_FORMAT_NAME" + ], + "last_modified" : 1422435345362, + "fact_table" : "SNOWTEST.KYLIN_SALES", + "filter_condition" : null, + "partition_desc" : { + "partition_date_column" : "SNOWTEST.KYLIN_SALES.PART_DT", + "partition_date_start" : 1325376000000, + "partition_type" : "APPEND" + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json new file mode 100644 index 0000000..8d39cbc --- /dev/null +++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json @@ -0,0 +1,28 @@ +{ + "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231", + + "name" : "KYLIN_ACCOUNT", + "columns" : [ { + "id" : "1", + "name" : "ACCOUNT_ID", + "datatype" : "bigint" + }, { + "id" : "2", + "name" : "ACCOUNT_BUYER_LEVEL", + "datatype" : "int" + }, { + "id" : "3", + "name" : "ACCOUNT_SELLER_LEVEL", + "datatype" : "int" + }, { + "id" : "4", + "name" : "ACCOUNT_COUNTRY", + "datatype" : "string" + }, { + "id" : "5", + "name" : "ACCOUNT_CONTACT", + "datatype" : "string" + } ], + "database" : "SNOWTEST", + "last_modified" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json new file mode 100644 index 0000000..655d4e7 --- /dev/null +++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json @@ -0,0 +1,408 @@ +{ + "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a", + + "name" : "KYLIN_CAL_DT", + "columns" : [ { + "id" : "1", + "name" : "CAL_DT", + "datatype" : "date" + }, { + "id" : "2", + "name" : "YEAR_BEG_DT", + "datatype" : "date" + }, { + "id" : "3", + "name" : "QTR_BEG_DT", + "datatype" : "date" + }, { + "id" : "4", + "name" : "MONTH_BEG_DT", + "datatype" : "date" + }, { + "id" : "5", + "name" : "WEEK_BEG_DT", + "datatype" : "date" + }, { + "id" : "6", + "name" : "AGE_FOR_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "7", + "name" : "AGE_FOR_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "8", + "name" : "AGE_FOR_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "9", + "name" : "AGE_FOR_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "10", + "name" : "AGE_FOR_DT_ID", + "datatype" : "smallint" + }, { + "id" : "11", + "name" : "AGE_FOR_RTL_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "12", + "name" : "AGE_FOR_RTL_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "13", + "name" : "AGE_FOR_RTL_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "14", + "name" : "AGE_FOR_RTL_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "15", + "name" : "AGE_FOR_CS_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "16", + "name" : "DAY_OF_CAL_ID", + "datatype" : "int" + }, { + "id" : "17", + "name" : "DAY_OF_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "18", + "name" : "DAY_OF_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "19", + "name" : "DAY_OF_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "20", + "name" : "DAY_OF_WEEK_ID", + "datatype" : "int" + }, { + "id" : "21", + "name" : "WEEK_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "22", + "name" : "WEEK_OF_CAL_ID", + "datatype" : "int" + }, { + "id" : "23", + "name" : "MONTH_OF_QTR_ID", + "datatype" : "tinyint" + }, { + "id" : "24", + "name" : "MONTH_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "25", + "name" : "MONTH_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "26", + "name" : "QTR_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "27", + "name" : "QTR_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "28", + "name" : "YEAR_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "29", + "name" : "YEAR_END_DT", + "datatype" : "string" + }, { + "id" : "30", + "name" : "QTR_END_DT", + "datatype" : "string" + }, { + "id" : "31", + "name" : "MONTH_END_DT", + "datatype" : "string" + }, { + "id" : "32", + "name" : "WEEK_END_DT", + "datatype" : "string" + }, { + "id" : "33", + "name" : "CAL_DT_NAME", + "datatype" : "string" + }, { + "id" : "34", + "name" : "CAL_DT_DESC", + "datatype" : "string" + }, { + "id" : "35", + "name" : "CAL_DT_SHORT_NAME", + "datatype" : "string" + }, { + "id" : "36", + "name" : "YTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "37", + "name" : "QTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "38", + "name" : "MTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "39", + "name" : "WTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "40", + "name" : "SEASON_BEG_DT", + "datatype" : "string" + }, { + "id" : "41", + "name" : "DAY_IN_YEAR_COUNT", + "datatype" : "smallint" + }, { + "id" : "42", + "name" : "DAY_IN_QTR_COUNT", + "datatype" : "tinyint" + }, { + "id" : "43", + "name" : "DAY_IN_MONTH_COUNT", + "datatype" : "tinyint" + }, { + "id" : "44", + "name" : "DAY_IN_WEEK_COUNT", + "datatype" : "tinyint" + }, { + "id" : "45", + "name" : "RTL_YEAR_BEG_DT", + "datatype" : "string" + }, { + "id" : "46", + "name" : "RTL_QTR_BEG_DT", + "datatype" : "string" + }, { + "id" : "47", + "name" : "RTL_MONTH_BEG_DT", + "datatype" : "string" + }, { + "id" : "48", + "name" : "RTL_WEEK_BEG_DT", + "datatype" : "string" + }, { + "id" : "49", + "name" : "CS_WEEK_BEG_DT", + "datatype" : "string" + }, { + "id" : "50", + "name" : "CAL_DATE", + "datatype" : "string" + }, { + "id" : "51", + "name" : "DAY_OF_WEEK", + "datatype" : "string" + }, { + "id" : "52", + "name" : "MONTH_ID", + "datatype" : "string" + }, { + "id" : "53", + "name" : "PRD_DESC", + "datatype" : "string" + }, { + "id" : "54", + "name" : "PRD_FLAG", + "datatype" : "string" + }, { + "id" : "55", + "name" : "PRD_ID", + "datatype" : "string" + }, { + "id" : "56", + "name" : "PRD_IND", + "datatype" : "string" + }, { + "id" : "57", + "name" : "QTR_DESC", + "datatype" : "string" + }, { + "id" : "58", + "name" : "QTR_ID", + "datatype" : "string" + }, { + "id" : "59", + "name" : "QTR_IND", + "datatype" : "string" + }, { + "id" : "60", + "name" : "RETAIL_WEEK", + "datatype" : "string" + }, { + "id" : "61", + "name" : "RETAIL_YEAR", + "datatype" : "string" + }, { + "id" : "62", + "name" : "RETAIL_START_DATE", + "datatype" : "string" + }, { + "id" : "63", + "name" : "RETAIL_WK_END_DATE", + "datatype" : "string" + }, { + "id" : "64", + "name" : "WEEK_IND", + "datatype" : "string" + }, { + "id" : "65", + "name" : "WEEK_NUM_DESC", + "datatype" : "string" + }, { + "id" : "66", + "name" : "WEEK_BEG_DATE", + "datatype" : "string" + }, { + "id" : "67", + "name" : "WEEK_END_DATE", + "datatype" : "string" + }, { + "id" : "68", + "name" : "WEEK_IN_YEAR_ID", + "datatype" : "string" + }, { + "id" : "69", + "name" : "WEEK_ID", + "datatype" : "string" + }, { + "id" : "70", + "name" : "WEEK_BEG_END_DESC_MDY", + "datatype" : "string" + }, { + "id" : "71", + "name" : "WEEK_BEG_END_DESC_MD", + "datatype" : "string" + }, { + "id" : "72", + "name" : "YEAR_ID", + "datatype" : "string" + }, { + "id" : "73", + "name" : "YEAR_IND", + "datatype" : "string" + }, { + "id" : "74", + "name" : "CAL_DT_MNS_1YEAR_DT", + "datatype" : "string" + }, { + "id" : "75", + "name" : "CAL_DT_MNS_2YEAR_DT", + "datatype" : "string" + }, { + "id" : "76", + "name" : "CAL_DT_MNS_1QTR_DT", + "datatype" : "string" + }, { + "id" : "77", + "name" : "CAL_DT_MNS_2QTR_DT", + "datatype" : "string" + }, { + "id" : "78", + "name" : "CAL_DT_MNS_1MONTH_DT", + "datatype" : "string" + }, { + "id" : "79", + "name" : "CAL_DT_MNS_2MONTH_DT", + "datatype" : "string" + }, { + "id" : "80", + "name" : "CAL_DT_MNS_1WEEK_DT", + "datatype" : "string" + }, { + "id" : "81", + "name" : "CAL_DT_MNS_2WEEK_DT", + "datatype" : "string" + }, { + "id" : "82", + "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "83", + "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "84", + "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "85", + "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "86", + "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "87", + "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "88", + "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND", + "datatype" : "tinyint" + }, { + "id" : "89", + "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND", + "datatype" : "tinyint" + }, { + "id" : "90", + "name" : "RTL_MONTH_OF_RTL_YEAR_ID", + "datatype" : "string" + }, { + "id" : "91", + "name" : "RTL_QTR_OF_RTL_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "92", + "name" : "RTL_WEEK_OF_RTL_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "93", + "name" : "SEASON_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "94", + "name" : "YTM_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "95", + "name" : "YTQ_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "96", + "name" : "YTW_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "97", + "name" : "KYLIN_CAL_DT_CRE_DATE", + "datatype" : "string" + }, { + "id" : "98", + "name" : "KYLIN_CAL_DT_CRE_USER", + "datatype" : "string" + }, { + "id" : "99", + "name" : "KYLIN_CAL_DT_UPD_DATE", + "datatype" : "string" + }, { + "id" : "100", + "name" : "KYLIN_CAL_DT_UPD_USER", + "datatype" : "string" + } ], + "database" : "SNOWTEST", + "last_modified" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json new file mode 100644 index 0000000..abee6d1 --- /dev/null +++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json @@ -0,0 +1,152 @@ +{ + "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f", + + "name" : "KYLIN_CATEGORY_GROUPINGS", + "columns" : [ { + "id" : "1", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint" + }, { + "id" : "2", + "name" : "LEAF_CATEG_NAME", + "datatype" : "string" + }, { + "id" : "3", + "name" : "SITE_ID", + "datatype" : "int" + }, { + "id" : "4", + "name" : "CATEG_BUSN_MGR", + "datatype" : "string" + }, { + "id" : "5", + "name" : "CATEG_BUSN_UNIT", + "datatype" : "string" + }, { + "id" : "6", + "name" : "REGN_CATEG", + "datatype" : "string" + }, { + "id" : "7", + "name" : "USER_DEFINED_FIELD1", + "datatype" : "string" + }, { + "id" : "8", + "name" : "USER_DEFINED_FIELD3", + "datatype" : "string" + }, { + "id" : "9", + "name" : "KYLIN_GROUPINGS_CRE_DATE", + "datatype" : "string" + }, { + "id" : "10", + "name" : "KYLIN_GROUPINGS_UPD_DATE", + "datatype" : "string" + }, { + "id" : "11", + "name" : "KYLIN_GROUPINGS_CRE_USER", + "datatype" : "string" + }, { + "id" : "12", + "name" : "KYLIN_GROUPINGS_UPD_USER", + "datatype" : "string" + }, { + "id" : "13", + "name" : "META_CATEG_ID", + "datatype" : "decimal" + }, { + "id" : "14", + "name" : "META_CATEG_NAME", + "datatype" : "string" + }, { + "id" : "15", + "name" : "CATEG_LVL2_ID", + "datatype" : "decimal" + }, { + "id" : "16", + "name" : "CATEG_LVL3_ID", + "datatype" : "decimal" + }, { + "id" : "17", + "name" : "CATEG_LVL4_ID", + "datatype" : "decimal" + }, { + "id" : "18", + "name" : "CATEG_LVL5_ID", + "datatype" : "decimal" + }, { + "id" : "19", + "name" : "CATEG_LVL6_ID", + "datatype" : "decimal" + }, { + "id" : "20", + "name" : "CATEG_LVL7_ID", + "datatype" : "decimal" + }, { + "id" : "21", + "name" : "CATEG_LVL2_NAME", + "datatype" : "string" + }, { + "id" : "22", + "name" : "CATEG_LVL3_NAME", + "datatype" : "string" + }, { + "id" : "23", + "name" : "CATEG_LVL4_NAME", + "datatype" : "string" + }, { + "id" : "24", + "name" : "CATEG_LVL5_NAME", + "datatype" : "string" + }, { + "id" : "25", + "name" : "CATEG_LVL6_NAME", + "datatype" : "string" + }, { + "id" : "26", + "name" : "CATEG_LVL7_NAME", + "datatype" : "string" + }, { + "id" : "27", + "name" : "CATEG_FLAGS", + "datatype" : "decimal" + }, { + "id" : "28", + "name" : "ADULT_CATEG_YN", + "datatype" : "string" + }, { + "id" : "29", + "name" : "DOMAIN_ID", + "datatype" : "decimal" + }, { + "id" : "30", + "name" : "USER_DEFINED_FIELD5", + "datatype" : "string" + }, { + "id" : "31", + "name" : "VCS_ID", + "datatype" : "decimal" + }, { + "id" : "32", + "name" : "GCS_ID", + "datatype" : "decimal" + }, { + "id" : "33", + "name" : "MOVE_TO", + "datatype" : "decimal" + }, { + "id" : "34", + "name" : "SAP_CATEGORY_ID", + "datatype" : "decimal" + }, { + "id" : "35", + "name" : "SRC_ID", + "datatype" : "tinyint" + }, { + "id" : "36", + "name" : "BSNS_VRTCL_NAME", + "datatype" : "string" + } ], + "database" : "SNOWTEST", + "last_modified" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json new file mode 100644 index 0000000..64708c7 --- /dev/null +++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json @@ -0,0 +1,24 @@ +{ + "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882", + + "name" : "KYLIN_COUNTRY", + "columns" : [ { + "id" : "1", + "name" : "COUNTRY", + "datatype" : "string" + }, { + "id" : "2", + "name" : "LATITUDE", + "datatype" : "double" + }, { + "id" : "3", + "name" : "LONGITUDE", + "datatype" : "double" + }, { + "id" : "4", + "name" : "NAME", + "datatype" : "string" + } ], + "database" : "SNOWTEST", + "last_modified" : 0 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json new file mode 100644 index 0000000..4aa82b5 --- /dev/null +++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json @@ -0,0 +1,56 @@ +{ + "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771", + + "name" : "KYLIN_SALES", + "columns" : [ { + "id" : "1", + "name" : "TRANS_ID", + "datatype" : "bigint" + }, { + "id" : "2", + "name" : "PART_DT", + "datatype" : "date" + }, { + "id" : "3", + "name" : "LSTG_FORMAT_NAME", + "datatype" : "string" + }, { + "id" : "4", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint" + }, { + "id" : "5", + "name" : "LSTG_SITE_ID", + "datatype" : "int" + }, { + "id" : "6", + "name" : "SLR_SEGMENT_CD", + "datatype" : "smallint" + }, { + "id" : "7", + "name" : "PRICE", + "datatype" : "decimal(19,4)" + }, { + "id" : "8", + "name" : "ITEM_COUNT", + "datatype" : "bigint" + }, { + "id" : "9", + "name" : "SELLER_ID", + "datatype" : "bigint" + }, { + "id" : "10", + "name" : "BUYER_ID", + "datatype" : "bigint" + }, { + "id" : "11", + "name" : "OPS_USER_ID", + "datatype" : "string" + }, { + "id" : "12", + "name" : "OPS_REGION", + "datatype" : "string" + } ], + "database" : "SNOWTEST", + "last_modified" : 0 +} http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java index 615d802..5fb2c5c 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java @@ -33,6 +33,7 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.JoinsTree; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.SQLDigest; @@ -127,6 +128,7 @@ public class OLAPContext { public Set<TblColRef> filterColumns = new HashSet<>(); public TupleFilter filter; public List<JoinDesc> joins = new LinkedList<>(); + public JoinsTree joinsTree; private List<TblColRef> sortColumns; private List<SQLDigest.OrderEnum> sortOrders; http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index 3c2bf48..14758c9 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -239,10 +239,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { return new ColumnRowType(columns); } + public TableRef getTableRef() { + return columnRowType.getColumnByIndex(0).getTableRef(); + } + @SuppressWarnings("deprecation") public TblColRef makeRewriteColumn(String name) { - TableRef tableRef = columnRowType.getColumnByIndex(0).getTableRef(); - return tableRef.makeFakeColumn(name); + return getTableRef().makeFakeColumn(name); } public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) { http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java index f1f5a48..ccb2900 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java +++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java @@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.JoinTableDesc; +import org.apache.kylin.metadata.model.JoinsTree; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectManager; @@ -38,7 +39,7 @@ import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.relnode.OLAPTableScan; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; -import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -73,47 +74,31 @@ public class ModelChooser { private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) { Map<String, String> result = Maps.newHashMap(); - // the greedy match is not perfect but works for the moment - Map<String, List<JoinDesc>> modelJoinsMap = model.getFKSideJoinMap(); for (OLAPContext ctx : contexts) { - for (JoinDesc queryJoin : ctx.joins) { - String fkTable = queryJoin.getForeignKeyColumns()[0].getTable(); - List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable); - if (modelJoins == null) - return null; - - JoinDesc matchJoin = null; - for (JoinDesc modelJoin : modelJoins) { - if (modelJoin.matches(queryJoin)) { - matchJoin = modelJoin; - break; - } - } - if (matchJoin == null) - return null; + TableRef firstTable = ctx.firstTableScan.getTableRef(); - matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result); - matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result); - } - - OLAPTableScan firstTable = ctx.firstTableScan; - String firstTableAlias = firstTable.getAlias(); - if (result.containsKey(firstTableAlias) == false) { - TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName()); - if (tableRef == null) - return null; - matchesAdd(firstTableAlias, tableRef.getAlias(), result); + Map<String, String> matchUp = null; + + if (ctx.joins.isEmpty() && model.isLookupTable(firstTable.getTableIdentity())) { + // one lookup table + String modelAlias = model.findFirstTable(firstTable.getTableIdentity()).getAlias(); + matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias); + } else { + // normal big joins + if (ctx.joinsTree == null) { + ctx.joinsTree = new JoinsTree(firstTable, ctx.joins); + } + matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result); } + + if (matchUp == null) + return null; + + result.putAll(matchUp); } - return result; } - private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) { - String existingTarget = result.put(origAlias, targetAlias); - Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias)); - } - private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) { // the first context, which is the top most context, contains all columns from all contexts OLAPContext first = contexts.get(0);
