Repository: kylin Updated Branches: refs/heads/KYLIN-1875 cdc618006 -> cc77ece63 (forced update)
KYLIN-1875 Normalize table and column names in JSON Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc77ece6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc77ece6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc77ece6 Branch: refs/heads/KYLIN-1875 Commit: cc77ece63741e8d6353bfb83aca76f4ac7c41720 Parents: bda8787 Author: Li Yang <liy...@apache.org> Authored: Tue Nov 29 20:32:04 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Tue Nov 29 21:20:03 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/model/CubeDesc.java | 2 ++ .../apache/kylin/cube/model/DimensionDesc.java | 19 +++++----- .../model/validation/rule/FunctionRule.java | 37 ++++---------------- .../kylin/metadata/model/DataModelDesc.java | 13 ++++++- .../kylin/metadata/model/FunctionDesc.java | 5 +-- .../metadata/model/ModelDimensionDesc.java | 19 +++++++--- .../kylin/metadata/model/PartitionDesc.java | 4 +-- 7 files changed, 48 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 47063d0..fa4da92 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -530,6 +530,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { initMeasureColumns(); rowkey.init(this); + validateAggregationGroups(); // check if aggregation group is valid for (AggregationGroup agg : this.aggregationGroups) { agg.init(this, rowkey); @@ -818,6 +819,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return col; } + @SuppressWarnings("deprecation") private void initMeasureColumns() { if (measures == null || measures.isEmpty()) { return; http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java index 1106103..cd75228 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java @@ -21,7 +21,6 @@ package org.apache.kylin.cube.model; import java.util.Arrays; import org.apache.commons.lang.NotImplementedException; -import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.JoinTableDesc; @@ -54,16 +53,13 @@ public class DimensionDesc { private TblColRef[] columnRefs; public void init(CubeDesc cubeDesc) { + DataModelDesc model = cubeDesc.getModel(); + if (name != null) name = name.toUpperCase(); - if (table != null) - table = table.toUpperCase(); - - DataModelDesc model = cubeDesc.getModel(); - tableRef = model.findTable(this.getTable()); - if (tableRef == null) - throw new IllegalStateException("Can't find table " + table + " for dimension " + name); + tableRef = model.findTable(table); + table = tableRef.getAlias(); join = null; for (JoinTableDesc joinTable : model.getJoinTables()) { @@ -73,11 +69,16 @@ public class DimensionDesc { } } + if (column != null && !"{FK}".equals(column)) { + column = model.findColumn(table, column).getName(); + } if (derived != null && derived.length == 0) { derived = null; } if (derived != null) { - StringUtil.toUpperCaseArray(derived, derived); + for (int i = 0; i < derived.length; i++) { + derived[i] = model.findColumn(table, derived[i]).getName(); + } } if (derived != null && join == null) { throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this); http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java index 1ed7325..36631ce 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java @@ -31,12 +31,10 @@ import org.apache.kylin.cube.model.validation.IValidatorRule; import org.apache.kylin.cube.model.validation.ResultLevel; import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.measure.topn.TopNMeasureType; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.ColumnDesc; +import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ParameterDesc; -import org.apache.kylin.metadata.model.TableDesc; import com.google.common.collect.Lists; @@ -153,35 +151,12 @@ public class FunctionRule implements IValidatorRule<CubeDesc> { * @param value */ private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) { - String factTable = cube.getModel().getRootFactTable().getTableIdentity(); - if (StringUtils.isEmpty(factTable)) { - context.addResult(ResultLevel.ERROR, "Fact table can not be null."); - return; - } - TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable); - if (table == null) { - context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube); - return; - } - // Prepare column set - Set<String> set = new HashSet<String>(); - ColumnDesc[] cdesc = table.getColumns(); - for (int i = 0; i < cdesc.length; i++) { - ColumnDesc columnDesc = cdesc[i]; - set.add(columnDesc.getName()); - } - - String[] items = value.split(","); - for (int i = 0; i < items.length; i++) { - String item = items[i].trim(); - if (StringUtils.isEmpty(item)) { - continue; - } - if (!set.contains(item)) { - context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable); - } + DataModelDesc model = cube.getModel(); + try { + model.findColumn(value); + } catch (IllegalArgumentException e) { + context.addResult(ResultLevel.ERROR, e.getMessage()); } - } /** http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index d917571..898ff74 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -282,7 +282,7 @@ public class DataModelDesc extends RootPersistentEntity { initTableAlias(tables); initJoinColumns(); initJoinsTree(); - ModelDimensionDesc.capicalizeStrings(dimensions); + initDimensionsAndMetrics(); initPartitionDesc(); } @@ -354,6 +354,15 @@ public class DataModelDesc extends RootPersistentEntity { } } + private void initDimensionsAndMetrics() { + for (ModelDimensionDesc dim : dimensions) { + dim.init(this); + } + for (int i = 0; i < metrics.length; i++) { + metrics[i] = findColumn(metrics[i]).getIdentity(); + } + } + private void initPartitionDesc() { if (this.partitionDesc != null) this.partitionDesc.init(this); @@ -381,6 +390,7 @@ public class DataModelDesc extends RootPersistentEntity { if (col == null || col.getTableRef().equals(dimTable) == false) { throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity()); } + pks[i] = col.getIdentity(); pkCols[i] = col; } join.setPrimaryKeyColumns(pkCols); @@ -393,6 +403,7 @@ public class DataModelDesc extends RootPersistentEntity { if (col == null) { throw new IllegalStateException("Can't find column " + fks[i] + " in table " + this.getRootFactTable()); } + fks[i] = col.getIdentity(); fkCols[i] = col; } join.setForeignKeyColumns(fkCols); http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index ae7f805..5818791 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -77,14 +77,11 @@ public class FunctionDesc { expression = expression.toUpperCase(); returnDataType = DataType.getType(returnType); - for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) { - p.setValue(p.getValue().toUpperCase()); - } - ArrayList<TblColRef> colRefs = Lists.newArrayList(); for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) { if (p.isColumnType()) { TblColRef colRef = model.findColumn(p.getValue()); + p.setValue(colRef.getIdentity()); colRefs.add(colRef); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java index d196155..6460f71 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java @@ -50,13 +50,24 @@ public class ModelDimensionDesc { this.columns = columns; } + void init(DataModelDesc model) { + table = table.toUpperCase(); + if (columns != null) { + StringUtil.toUpperCaseArray(columns, columns); + } + + if (model != null) { + table = model.findTable(table).getAlias(); + for (int i = 0; i < columns.length; i++) { + columns[i] = model.findColumn(table, columns[i]).getName(); + } + } + } + public static void capicalizeStrings(List<ModelDimensionDesc> dimensions) { if (dimensions != null) { for (ModelDimensionDesc modelDimensionDesc : dimensions) { - modelDimensionDesc.setTable(modelDimensionDesc.getTable().toUpperCase()); - if (modelDimensionDesc.getColumns() != null) { - StringUtil.toUpperCaseArray(modelDimensionDesc.getColumns(), modelDimensionDesc.getColumns()); - } + modelDimensionDesc.init(null); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cc77ece6/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 0261f41..9925990 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -66,11 +66,11 @@ public class PartitionDesc { if (StringUtils.isEmpty(partitionDateColumn)) return; - partitionDateColumn = partitionDateColumn.toUpperCase(); partitionDateColumnRef = model.findColumn(partitionDateColumn); + partitionDateColumn = partitionDateColumnRef.getIdentity(); if (StringUtils.isBlank(partitionTimeColumn) == false) { - partitionTimeColumn = partitionTimeColumn.toUpperCase(); partitionTimeColumnRef = model.findColumn(partitionTimeColumn); + partitionTimeColumn = partitionTimeColumnRef.getIdentity(); } partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz); }
