tmp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1e6ff68 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1e6ff68 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1e6ff68 Branch: refs/heads/KYLIN-1971 Commit: c1e6ff6807097324a8c303796217a34c37ad08a2 Parents: 5cdd225 Author: Li Yang <liy...@apache.org> Authored: Wed Oct 26 18:24:15 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Wed Oct 26 18:24:15 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeInstance.java | 6 ++++++ .../java/org/apache/kylin/cube/model/CubeDesc.java | 14 ++++++++++++-- .../org/apache/kylin/metadata/model/JoinDesc.java | 2 +- .../org/apache/kylin/metadata/model/TblColRef.java | 4 ++-- .../kylin/metadata/realization/IRealization.java | 3 +++ .../kylin/storage/hybrid/HybridInstance.java | 17 +++++++++++++++++ .../org/apache/kylin/query/ITKylinQueryTest.java | 6 +++--- .../apache/kylin/query/routing/ModelChooser.java | 12 +++++++++++- 8 files changed, 55 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 2ccdde7..6bd3826 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.LookupDesc; @@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } @Override + public Set<ColumnDesc> getAllColumnDescs() { + return getDescriptor().listAllColumnDescs(); + } + + @Override public long getDateRangeStart() { List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY); http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 5c73f21..0c0b4f3 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; @@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @JsonProperty("override_kylin_properties") private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>(); - private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>(); - private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>(); + private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>(); + private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>(); + private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>(); private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap(); private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); @@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public Set<TblColRef> listAllColumns() { return allColumns; } + + public Set<ColumnDesc> listAllColumnDescs() { + return allColumnDescs; + } /** * @return dimension columns including derived, BUT NOT measures @@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%d) doesn't match dimensions columns count (%d)", rowkey.getRowKeyColumns().length, dimCols.size()); initDictionaryDesc(); + + for (TblColRef col : allColumns) { + allColumnDescs.add(col.getColumnDesc()); + } } public void validateAggregationGroups() { http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java index 04fbf62..5beda0a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java @@ -154,7 +154,7 @@ public class JoinDesc { int match = 0; for (int i = 0; i < a.length; i++) { for (int j = 0; j < b.length; j++) { - if (a[i].equals(b[j])) { + if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) { match++; break; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index 8841ee5..f44972c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -183,8 +183,8 @@ public class TblColRef implements Serializable { return false; if (!StringUtils.equals(column.getName(), other.column.getName())) return false; -// if ((table == null ? other.table == null : table.equals(other.table)) == false) -// return false; + if ((table == null ? other.table == null : table.equals(other.table)) == false) + return false; return true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index 343ec99..f1770d7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization; import java.util.List; import java.util.Set; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.MeasureDesc; @@ -43,6 +44,8 @@ public interface IRealization extends IStorageAware { public String getFactTable(); public Set<TblColRef> getAllColumns(); + + public Set<ColumnDesc> getAllColumnDescs(); public List<TblColRef> getAllDimensions(); http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 57d14d5..d7007ac 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -68,6 +69,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization private volatile IRealization[] realizations = null; private List<TblColRef> allDimensions = null; private Set<TblColRef> allColumns = null; + private Set<ColumnDesc> allColumnDescs = null; private List<MeasureDesc> allMeasures = null; private long dateRangeStart; private long dateRangeEnd; @@ -138,6 +140,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization allDimensions = Lists.newArrayList(dimensions); allColumns = columns; + allColumnDescs = asColumnDescs(allColumns); allMeasures = Lists.newArrayList(measures); Collections.sort(realizationList, new Comparator<IRealization>() { @@ -166,6 +169,14 @@ public class HybridInstance extends RootPersistentEntity implements IRealization } } + private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) { + LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>(); + for (TblColRef col : columns) { + result.add(col.getColumnDesc()); + } + return result; + } + @Override public CapabilityResult isCapable(SQLDigest digest) { CapabilityResult result = new CapabilityResult(); @@ -210,6 +221,12 @@ public class HybridInstance extends RootPersistentEntity implements IRealization } @Override + public Set<ColumnDesc> getAllColumnDescs() { + init(); + return allColumnDescs; + } + + @Override public List<MeasureDesc> getMeasures() { init(); return allMeasures; http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 2ec5324..54f02aa 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -45,7 +45,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; -@Ignore("KylinQueryTest is contained by ITCombinationTest") +//@Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { @Rule @@ -150,11 +150,11 @@ public class ITKylinQueryTest extends KylinTestBase { } - @Ignore + //@Ignore @Test public void testSingleRunQuery() throws Exception { - String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql"; + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java index 4667f4a..fcfcfbb 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java +++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java @@ -26,10 +26,12 @@ import java.util.TreeMap; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.LookupDesc; import org.apache.kylin.metadata.model.TableRef; +import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.relnode.OLAPContext; @@ -110,7 +112,7 @@ public class ModelChooser { for (IRealization real : realizations) { if (real.isReady() == false) continue; - if (real.getAllColumns().containsAll(context.allColumns) == false) + if (containsAll(real.getAllColumnDescs(), context.allColumns) == false) continue; if (RemoveBlackoutRealizationsRule.accept(real) == false) continue; @@ -143,6 +145,14 @@ public class ModelChooser { return result; } + private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) { + for (TblColRef col : allColumns) { + if (allColumnDescs.contains(col.getColumnDesc()) == false) + return false; + } + return true; + } + private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) { for (OLAPTableScan tableScan : context.allTableScans) { tableScan.fixColumnRowTypeWithModel(model, aliasMap);
