Repository: kylin Updated Branches: refs/heads/KYLIN-1875 a9100bfc6 -> 48253d1ff
KYLIN-1875 fix PK-FK derive from each other Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48253d1f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48253d1f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48253d1f Branch: refs/heads/KYLIN-1875 Commit: 48253d1ff03e137a8568b8ca880c45b5b8c524f3 Parents: a9100bf Author: Yang Li <liy...@apache.org> Authored: Thu Dec 1 21:42:52 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Thu Dec 1 21:42:52 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeManager.java | 8 +-- .../org/apache/kylin/cube/model/CubeDesc.java | 52 +++++++++++--------- .../storage/gtrecord/CubeTupleConverter.java | 2 +- .../gtrecord/GTCubeStorageQueryBase.java | 2 +- .../translate/DerivedFilterTranslator.java | 2 +- .../apache/kylin/query/ITKylinQueryTest.java | 2 +- .../query/enumerator/LookupTableEnumerator.java | 2 +- .../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +- .../hbase/cube/v1/CubeTupleConverter.java | 2 +- 9 files changed, 41 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index b4422d2..6a25594 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -45,7 +45,6 @@ import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.DictionaryDesc; -import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.dict.DictionaryInfo; import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.dict.lookup.LookupStringTable; @@ -55,6 +54,7 @@ import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; +import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -657,10 +657,10 @@ public class CubeManager implements IRealizationProvider { cubeMap.removeLocal(cubeName); } - public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { + public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) { - String tableName = dim.getTableRef().getTableIdentity(); - String[] pkCols = dim.getJoin().getPrimaryKey(); + String tableName = join.getPKSide().getTableIdentity(); + String[] pkCols = join.getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index fa4da92..18b7e7a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; @@ -98,20 +99,20 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public static class DeriveInfo { public DeriveType type; - public DimensionDesc dimension; + public JoinDesc join; public TblColRef[] columns; public boolean isOneToOne; // only used when ref from derived to host - DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) { + DeriveInfo(DeriveType type, JoinDesc join, TblColRef[] columns, boolean isOneToOne) { this.type = type; - this.dimension = dimension; + this.join = join; this.columns = columns; this.isOneToOne = isOneToOne; } @Override public String toString() { - return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; + return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; } } @@ -729,27 +730,34 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { for (int i = 0; i < derivedNames.length; i++) { derivedCols[i] = initDimensionColRef(dim, derivedNames[i]); } - initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra); + initDerivedMap(dimColArray, DeriveType.LOOKUP, join, derivedCols, derivedExtra); } - - // PK-FK derive the other side + if (join != null) { - TblColRef[] fk = join.getForeignKeyColumns(); - TblColRef[] pk = join.getPrimaryKeyColumns(); - - allColumns.addAll(Arrays.asList(fk)); - allColumns.addAll(Arrays.asList(pk)); - for (int i = 0; i < fk.length; i++) { - int find = ArrayUtils.indexOf(dimColArray, fk[i]); - if (find >= 0) { - TblColRef derivedCol = initDimensionColRef(pk[i]); - initDerivedMap(new TblColRef[] { dimColArray[find] }, DeriveType.PK_FK, dim, new TblColRef[] { derivedCol }, null); - } + allColumns.addAll(Arrays.asList(join.getForeignKeyColumns())); + allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns())); + } + } + + // PK-FK derive the other side + Set<TblColRef> realDimensions = new HashSet<>(listDimensionColumnsExcludingDerived(true)); + for (JoinTableDesc joinTable : model.getJoinTables()) { + JoinDesc join = joinTable.getJoin(); + int n = join.getForeignKeyColumns().length; + for (int i = 0; i < n; i++) { + TblColRef pk = join.getPrimaryKeyColumns()[i]; + TblColRef fk = join.getForeignKeyColumns()[i]; + if (realDimensions.contains(pk) && !realDimensions.contains(fk)) { + initDimensionColRef(fk); + initDerivedMap(new TblColRef[] { pk }, DeriveType.PK_FK, join, new TblColRef[] { fk }, null); + } else if (realDimensions.contains(fk) && !realDimensions.contains(pk)) { + initDimensionColRef(pk); + initDerivedMap(new TblColRef[] { fk }, DeriveType.PK_FK, join, new TblColRef[] { pk }, null); } } } } - + private String[][] splitDerivedColumnAndExtra(String[] derived) { String[] cols = new String[derived.length]; String[] extra = new String[derived.length]; @@ -767,7 +775,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return new String[][] { cols, extra }; } - private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { + private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); @@ -790,12 +798,12 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { if (infoList == null) { hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); } - infoList.add(new DeriveInfo(type, dimension, derivedCols, false)); + infoList.add(new DeriveInfo(type, join, derivedCols, false)); for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); - toHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne)); + toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java index 7ec24b2..3159318 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java @@ -266,7 +266,7 @@ public class CubeTupleConverter { case LOOKUP: return new IDerivedColumnFiller() { CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig()); - LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension); + LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join); int[] derivedColIdx = initDerivedColIdx(); Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]); http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 9c74cca..31573d0 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -294,7 +294,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { DeriveInfo hostInfo = cubeDesc.getHostInfo(derived); CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig()); CubeSegment seg = cubeInstance.getLatestReadySegment(); - LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension); + LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join); Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf); TupleFilter translatedFilter = translated.getFirst(); boolean loosened = translated.getSecond(); http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java index 160338f..13c655c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java @@ -54,7 +54,7 @@ public class DerivedFilterTranslator { TblColRef derivedCol = compf.getColumn(); TblColRef[] hostCols = hostInfo.columns; - TblColRef[] pkCols = hostInfo.dimension.getJoin().getPrimaryKeyColumns(); + TblColRef[] pkCols = hostInfo.join.getPrimaryKeyColumns(); if (hostInfo.type == DeriveType.PK_FK) { assert hostCols.length == 1; http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 90324b5..6d91753 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; -@Ignore("KylinQueryTest is contained by ITCombinationTest") +//@Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { @Rule http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java index 1c2bc2d..28ee623 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java @@ -67,7 +67,7 @@ public class LookupTableEnumerator implements Enumerator<Object[]> { throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor()); CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig()); - LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim); + LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim.getJoin()); this.allRows = table.getAllRows(); OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable(); http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 9af0faf..ca46188 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -368,7 +368,7 @@ public class CubeStorageQuery implements IStorageQuery { DeriveInfo hostInfo = cubeDesc.getHostInfo(derived); CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig()); CubeSegment seg = cubeInstance.getLatestReadySegment(); - LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension); + LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join); Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf); TupleFilter translatedFilter = translated.getFirst(); boolean loosened = translated.getSecond(); http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java index d2378b9..64feff0 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java @@ -220,7 +220,7 @@ public class CubeTupleConverter { case LOOKUP: return new IDerivedColumnFiller() { CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig()); - LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension); + LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join); int[] derivedColIdx = initDerivedColIdx(); Array<String> lookupKey = new Array<String>(new String[hostColIdx.length]);
