KYLIN-2423 Model should always include PK/FK as dimensions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/855301dc Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/855301dc Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/855301dc Branch: refs/heads/master-hbase0.98 Commit: 855301dc7fc83ece4f03dd981f3184bb46a97105 Parents: 56c7aa5 Author: Li Yang <liy...@apache.org> Authored: Sat Feb 4 15:19:31 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sat Feb 4 16:12:42 2017 +0800 ---------------------------------------------------------------------- .../kylin/metadata/model/DataModelDesc.java | 76 ++++++++++++++++++++ .../kylin/metadata/model/JoinTableDesc.java | 7 +- .../apache/kylin/metadata/model/JoinsTree.java | 11 +-- .../metadata/model/ModelDimensionDesc.java | 2 + .../model_desc/ut_large_dimension_number.json | 2 +- 5 files changed, 90 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index cc599a4..86eea55 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -20,15 +20,18 @@ package org.apache.kylin.metadata.model; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.ArrayUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.MetadataConstants; +import org.apache.kylin.metadata.model.JoinsTree.Chain; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -294,6 +297,11 @@ public class DataModelDesc extends RootPersistentEntity { initJoinsTree(); initDimensionsAndMetrics(); initPartitionDesc(); + + boolean reinit = validate(); + if (reinit) { // model slightly changed by validate() and must init() again + init(config, tables); + } } private void initJoinTablesForUpgrade() { @@ -443,6 +451,74 @@ public class DataModelDesc extends RootPersistentEntity { joinsTree = new JoinsTree(rootFactTableRef, joins); } + private boolean validate() { + Set<TblColRef> mcols = new HashSet<>(); + for (String m : metrics) { + mcols.add(findColumn(m)); + } + + // validate no dup between dimensions/metrics + for (ModelDimensionDesc dim : dimensions) { + String table = dim.getTable(); + for (String c : dim.getColumns()) { + TblColRef dcol = findColumn(table, c); + if (mcols.contains(dcol)) + throw new IllegalStateException(dcol + " cannot be both dimension and metrics at the same time in " + this); + } + } + + // validate PK/FK are in dimensions + boolean pkfkDimAmended = false; + for (Chain chain : joinsTree.tableChains.values()) { + pkfkDimAmended = validatePkFkDim(chain.join, mcols) || pkfkDimAmended; + } + return pkfkDimAmended; + } + + private boolean validatePkFkDim(JoinDesc join, Set<TblColRef> mcols) { + if (join == null) + return false; + + boolean pkfkDimAmended = false; + + for (TblColRef c : join.getForeignKeyColumns()) { + if (!mcols.contains(c)) { + pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended; + } + } + for (TblColRef c : join.getPrimaryKeyColumns()) { + if (!mcols.contains(c)) { + pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended; + } + } + return pkfkDimAmended; + } + + private boolean validatePkFkDim(TblColRef c) { + String t = c.getTableAlias(); + ModelDimensionDesc dimDesc = null; + for (ModelDimensionDesc dim : dimensions) { + if (dim.getTable().equals(t)) { + dimDesc = dim; + break; + } + } + + if (dimDesc == null) { + dimDesc = new ModelDimensionDesc(); + dimDesc.setTable(t); + dimDesc.setColumns(new String[0]); + } + + if (ArrayUtils.contains(dimDesc.getColumns(), c.getName()) == false) { + String[] newCols = ArrayUtils.add(dimDesc.getColumns(), c.getName()); + dimDesc.setColumns(newCols); + return true; + } + + return false; + } + /** * Add error info and thrown exception out */ http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java index 51e5787..9ca806e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java @@ -18,17 +18,18 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; + import org.apache.kylin.metadata.model.DataModelDesc.TableKind; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.Serializable; - @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class JoinTableDesc implements Serializable { + private static final long serialVersionUID = 1L; @JsonProperty("table") private String table; http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java index c132d0e..3ab9a46 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java @@ -18,8 +18,6 @@ package org.apache.kylin.metadata.model; -import com.google.common.base.Preconditions; - import java.io.Serializable; import java.util.Collections; import java.util.HashMap; @@ -27,8 +25,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -public class JoinsTree implements Serializable { +import com.google.common.base.Preconditions; +public class JoinsTree implements Serializable { + private static final long serialVersionUID = 1L; + final Map<String, Chain> tableChains = new LinkedHashMap<>(); public JoinsTree(TableRef rootTable, List<JoinDesc> joins) { @@ -112,7 +113,9 @@ public class JoinsTree implements Serializable { return chain.join; } - static class Chain implements java.io.Serializable { + static class Chain implements Serializable { + private static final long serialVersionUID = 1L; + TableRef table; // pk side JoinDesc join; Chain fkSide; http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java index d14a56b..c0ddbad 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java @@ -30,6 +30,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; */ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) public class ModelDimensionDesc implements Serializable { + private static final long serialVersionUID = 1L; + @JsonProperty("table") private String table; @JsonProperty("columns") http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json b/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json index 38b070e..9eeb774 100644 --- a/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json +++ b/examples/test_case_data/localmeta/model_desc/ut_large_dimension_number.json @@ -11,7 +11,7 @@ "table" : "DEFAULT.WIDE_TABLE", "columns" : ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA", "AB", "AC", "AD", "AE", "AF", "AG", "AH", "AI", "AJ", "AK", "AL", "AM", "AN", "AO", "AP", "AQ", "AR", "AS", "AT", "AU", "AV", "AW", "AX", "AY", "AZ"] }], - "metrics" : [ "A" ], + "metrics" : [ ], "filter_condition" : "", "partition_desc" : { "partition_date_column" : "DEFAULT.WIDE_TABLE.A",
