KYLIN-2330 fix CubeDesc returning redundant DeriveInfo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a190ecf Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a190ecf Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a190ecf Branch: refs/heads/master-cdh5.7 Commit: 1a190ecfcaad88e2551c5aff3bd4938719ba46a6 Parents: fdecf09 Author: Yang Li <liy...@apache.org> Authored: Thu Dec 29 07:13:12 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Thu Dec 29 07:31:17 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/QueryContext.java | 2 +- .../org/apache/kylin/cube/model/CubeDesc.java | 46 +++++++++++++++----- .../org/apache/kylin/cube/CubeDescTest.java | 45 ++++++++++++++++++- .../test_case_data/localmeta/cube_desc/ssb.json | 12 ++++- 4 files changed, 91 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-common/src/main/java/org/apache/kylin/common/QueryContext.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java index ef0cb14..93b8556 100644 --- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java +++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java @@ -23,7 +23,7 @@ import java.util.Map; import com.google.common.collect.Maps; /** - * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision + * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparison */ public class QueryContext { private static final ThreadLocal<Map<String, String>> _queryContext = new ThreadLocal<Map<String, String>>(); http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 3b8d034..4686a15 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -807,25 +807,49 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { for (int i = 0; i < derivedCols.length; i++) { if (ArrayUtils.contains(hostCols, derivedCols[i])) { derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i); - extra = (String[]) ArrayUtils.remove(extra, i); + if (extra != null) + extra = (String[]) ArrayUtils.remove(extra, i); i--; } } + + if (derivedCols.length == 0) + return; - Map<TblColRef, DeriveInfo> toHostMap = derivedToHostMap; - Map<Array<TblColRef>, List<DeriveInfo>> hostToMap = hostToDerivedMap; + for (int i = 0; i < derivedCols.length; i++) { + TblColRef derivedCol = derivedCols[i]; + boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); + derivedToHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne)); + } Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); - List<DeriveInfo> infoList = hostToMap.get(hostColArray); + List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); if (infoList == null) { - hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); + hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); } - infoList.add(new DeriveInfo(type, join, derivedCols, false)); - - for (int i = 0; i < derivedCols.length; i++) { - TblColRef derivedCol = derivedCols[i]; - boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); - toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne)); + + // Merged duplicated derived column + List<TblColRef> whatsLeft = new ArrayList<>(); + for (TblColRef derCol : derivedCols) { + boolean merged = false; + for (DeriveInfo existing : infoList) { + if (existing.type == type && existing.join.getPKSide().equals(join.getPKSide())) { + if (ArrayUtils.contains(existing.columns, derCol)) { + merged = true; + break; + } + if (type == DeriveType.LOOKUP) { + existing.columns = (TblColRef[]) ArrayUtils.add(existing.columns, derCol); + merged = true; + break; + } + } + } + if (!merged) + whatsLeft.add(derCol); + } + if (whatsLeft.size() > 0) { + infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]), false)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 0a7adf1..86ea1df 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -23,20 +23,28 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; +import org.apache.kylin.cube.model.CubeDesc.DeriveType; import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.cube.model.SelectRule; import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.TblColRef; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -331,8 +339,43 @@ public class CubeDescTest extends LocalFileMetadataTestCase { Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class); - assertEquals(map, map2); + Assert.assertEquals(map, map2); + } + @Test + public void testDerivedInfo() { + { + CubeDesc cube = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); + List<TblColRef> givenCols = new ArrayList<>(); + givenCols.add(cube.findColumnRef("TEST_KYLIN_FACT", "LSTG_SITE_ID")); + givenCols.add(cube.findColumnRef("TEST_KYLIN_FACT", "LEAF_CATEG_ID")); + Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cube.getHostToDerivedInfo(givenCols, null); + assertEquals(3, hostToDerivedInfo.size()); + assertEquals(Pair.newPair(3, 2), countDerivedInfo(hostToDerivedInfo)); + } + + { + CubeDesc cube = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ssb"); + List<TblColRef> givenCols = new ArrayList<>(); + givenCols.add(cube.findColumnRef("V_LINEORDER", "LO_PARTKEY")); + Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cube.getHostToDerivedInfo(givenCols, null); + assertEquals(1, hostToDerivedInfo.size()); + assertEquals(Pair.newPair(1, 1), countDerivedInfo(hostToDerivedInfo)); + } + } + + private Pair<Integer, Integer> countDerivedInfo(Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo) { + int pkfkCount = 0; + int lookupCount = 0; + for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) { + for (DeriveInfo deriveInfo : entry.getValue()) { + if (deriveInfo.type == DeriveType.PK_FK) + pkfkCount++; + if (deriveInfo.type == DeriveType.LOOKUP) + lookupCount++; + } + } + return Pair.newPair(pkfkCount, lookupCount); } private Collection<String> sortStrs(String[] strs) { http://git-wip-us.apache.org/repos/asf/kylin/blob/1a190ecf/examples/test_case_data/localmeta/cube_desc/ssb.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json index d3ea10b..a13ac53 100644 --- a/examples/test_case_data/localmeta/cube_desc/ssb.json +++ b/examples/test_case_data/localmeta/cube_desc/ssb.json @@ -6,7 +6,17 @@ "name" : "SSB.PART_DERIVED", "table" : "SSB.PART", "column" : null, - "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ] + "derived" : [ "P_MFGR" ] + }, { + "name" : "SSB.PART_DERIVED", + "table" : "SSB.PART", + "column" : null, + "derived" : [ "P_CATEGORY" ] + }, { + "name" : "SSB.PART_DERIVED", + "table" : "SSB.PART", + "column" : null, + "derived" : [ "P_BRAND" ] }, { "name" : "C_CITY", "table" : "SSB.CUSTOMER",
