TblColRef != ColumnDesc, turns on
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/293eb85b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/293eb85b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/293eb85b Branch: refs/heads/KYLIN-1971 Commit: 293eb85b63869f25d356309b28a0b6a3a55aad06 Parents: 69213e9 Author: Li Yang <liy...@apache.org> Authored: Wed Oct 26 18:24:15 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Thu Oct 27 07:15:53 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeInstance.java | 6 + .../org/apache/kylin/cube/model/CubeDesc.java | 14 ++- .../apache/kylin/metadata/model/JoinDesc.java | 2 +- .../apache/kylin/metadata/model/TblColRef.java | 4 +- .../metadata/realization/IRealization.java | 3 + .../kylin/storage/hybrid/HybridInstance.java | 17 +++ .../apache/kylin/query/ITKylinQueryTest.java | 2 +- .../relnode/OLAPToEnumerableConverter.java | 53 +++++---- .../kylin/query/routing/ModelChooser.java | 111 ++++++++++++------- 9 files changed, 139 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 2ccdde7..6bd3826 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IBuildable; import org.apache.kylin.metadata.model.LookupDesc; @@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } @Override + public Set<ColumnDesc> getAllColumnDescs() { + return getDescriptor().listAllColumnDescs(); + } + + @Override public long getDateRangeStart() { List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY); http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 5c73f21..0c0b4f3 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; @@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @JsonProperty("override_kylin_properties") private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>(); - private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>(); - private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>(); + private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>(); + private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>(); + private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>(); private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap(); private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); @@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public Set<TblColRef> listAllColumns() { return allColumns; } + + public Set<ColumnDesc> listAllColumnDescs() { + return allColumnDescs; + } /** * @return dimension columns including derived, BUT NOT measures @@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%d) doesn't match dimensions columns count (%d)", rowkey.getRowKeyColumns().length, dimCols.size()); initDictionaryDesc(); + + for (TblColRef col : allColumns) { + allColumnDescs.add(col.getColumnDesc()); + } } public void validateAggregationGroups() { http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java index 04fbf62..5beda0a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java @@ -154,7 +154,7 @@ public class JoinDesc { int match = 0; for (int i = 0; i < a.length; i++) { for (int j = 0; j < b.length; j++) { - if (a[i].equals(b[j])) { + if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) { match++; break; } http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index 8841ee5..f44972c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -183,8 +183,8 @@ public class TblColRef implements Serializable { return false; if (!StringUtils.equals(column.getName(), other.column.getName())) return false; -// if ((table == null ? other.table == null : table.equals(other.table)) == false) -// return false; + if ((table == null ? other.table == null : table.equals(other.table)) == false) + return false; return true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index 343ec99..f1770d7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization; import java.util.List; import java.util.Set; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.MeasureDesc; @@ -43,6 +44,8 @@ public interface IRealization extends IStorageAware { public String getFactTable(); public Set<TblColRef> getAllColumns(); + + public Set<ColumnDesc> getAllColumnDescs(); public List<TblColRef> getAllDimensions(); http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 57d14d5..d7007ac 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -68,6 +69,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization private volatile IRealization[] realizations = null; private List<TblColRef> allDimensions = null; private Set<TblColRef> allColumns = null; + private Set<ColumnDesc> allColumnDescs = null; private List<MeasureDesc> allMeasures = null; private long dateRangeStart; private long dateRangeEnd; @@ -138,6 +140,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization allDimensions = Lists.newArrayList(dimensions); allColumns = columns; + allColumnDescs = asColumnDescs(allColumns); allMeasures = Lists.newArrayList(measures); Collections.sort(realizationList, new Comparator<IRealization>() { @@ -166,6 +169,14 @@ public class HybridInstance extends RootPersistentEntity implements IRealization } } + private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) { + LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>(); + for (TblColRef col : columns) { + result.add(col.getColumnDesc()); + } + return result; + } + @Override public CapabilityResult isCapable(SQLDigest digest) { CapabilityResult result = new CapabilityResult(); @@ -210,6 +221,12 @@ public class HybridInstance extends RootPersistentEntity implements IRealization } @Override + public Set<ColumnDesc> getAllColumnDescs() { + init(); + return allColumnDescs; + } + + @Override public List<MeasureDesc> getMeasures() { init(); return allMeasures; http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 2ec5324..2065bff 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -154,7 +154,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { - String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql"; + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java index 24fc430..31cfc36 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java @@ -48,10 +48,10 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.routing.ModelChooser; -import org.apache.kylin.query.routing.NoRealizationFoundException; import org.apache.kylin.query.routing.QueryRouter; import org.apache.kylin.query.schema.OLAPTable; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** @@ -79,30 +79,23 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab OLAPRel.OLAPImplementor olapImplementor = new OLAPRel.OLAPImplementor(); olapImplementor.visitChild(getInput(), this); - // find cube from olap context and apply cell level security - try { - for (OLAPContext context : OLAPContext.getThreadLocalContexts()) { - // Context has no table scan is created by OLAPJoinRel which looks like - // (sub-query) as A join (sub-query) as B - // No realization needed for such context. - if (context.firstTableScan == null) { - continue; - } - - Set<IRealization> candidates = ModelChooser.selectModel(context); - IRealization realization = QueryRouter.selectRealization(context, candidates); - context.realization = realization; - - doAccessControl(context); - } - } catch (NoRealizationFoundException e) { - OLAPContext ctx0 = (OLAPContext) OLAPContext.getThreadLocalContexts().toArray()[0]; - if (ctx0 != null && ctx0.olapSchema.hasStarSchemaUrl()) { - // generate hive result - return buildHiveResult(enumImplementor, pref, ctx0); - } else { - throw e; + // identify model + List<OLAPContext> contexts = listContextsHavingScan(); + Set<IRealization> candidates = ModelChooser.selectModel(contexts); + + // identify realization for each context + for (OLAPContext context : OLAPContext.getThreadLocalContexts()) { + + // Context has no table scan is created by OLAPJoinRel which looks like + // (sub-query) as A join (sub-query) as B + // No realization needed for such context. + if (context.firstTableScan == null) { + continue; } + + IRealization realization = QueryRouter.selectRealization(context, candidates); + context.realization = realization; + doAccessControl(context); } // rewrite query if necessary @@ -123,6 +116,17 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab return impl.visitChild(this, 0, inputAsEnum, pref); } + private List<OLAPContext> listContextsHavingScan() { + int size = OLAPContext.getThreadLocalContexts().size(); + List<OLAPContext> result = Lists.newArrayListWithCapacity(size); + for (int i = 0; i < size; i++) { + OLAPContext ctx = OLAPContext.getThreadLocalContextById(i); + if (ctx.firstTableScan != null) + result.add(ctx); + } + return result; + } + private void doAccessControl(OLAPContext context) { String controllerCls = KylinConfig.getInstanceFromEnv().getQueryAccessController(); if (null != controllerCls && !controllerCls.isEmpty()) { @@ -176,6 +180,7 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab } } + @SuppressWarnings("unused") private Result buildHiveResult(EnumerableRelImplementor enumImplementor, Prefer pref, OLAPContext context) { RelDataType hiveRowType = getRowType(); http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java index 4667f4a..08616bd 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java +++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java @@ -26,10 +26,12 @@ import java.util.TreeMap; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.LookupDesc; import org.apache.kylin.metadata.model.TableRef; +import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.relnode.OLAPContext; @@ -42,55 +44,68 @@ import com.google.common.collect.Sets; public class ModelChooser { - public static Set<IRealization> selectModel(OLAPContext context) { - Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context); - OLAPTableScan firstTable = context.firstTableScan; - List<JoinDesc> joins = context.joins; + // select a model that satisfies all the contexts + public static Set<IRealization> selectModel(List<OLAPContext> contexts) { + Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(contexts); for (DataModelDesc model : modelMap.keySet()) { - Map<String, String> aliasMap = matches(model, firstTable, joins); + Map<String, String> aliasMap = matches(model, contexts); if (aliasMap != null) { - fixModel(context, model, aliasMap); + for (OLAPContext ctx : contexts) + fixModel(ctx, model, aliasMap); return modelMap.get(model); } } - - throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins); + + throw new NoRealizationFoundException("No model found for" + toErrorMsg(contexts)); } - private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) { - Map<String, String> result = Maps.newHashMap(); - - // no join special case - if (joins.isEmpty()) { - TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName()); - if (tableRef == null) - return null; - result.put(firstTable.getAlias(), tableRef.getAlias()); - return result; + private static String toErrorMsg(List<OLAPContext> contexts) { + StringBuilder buf = new StringBuilder(); + for (OLAPContext ctx : contexts) { + buf.append(", ").append(ctx.firstTableScan); + for (JoinDesc join : ctx.joins) + buf.append(", ").append(join); } - + return buf.toString(); + } + + private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) { + Map<String, String> result = Maps.newHashMap(); + // the greedy match is not perfect but works for the moment Map<String, List<JoinDesc>> modelJoinsMap = model.getJoinsMap(); - for (JoinDesc queryJoin : joins) { - String fkTable = queryJoin.getForeignKeyColumns()[0].getTable(); - List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable); - if (modelJoins == null) - return null; - - JoinDesc matchJoin = null; - for (JoinDesc modelJoin : modelJoins) { - if (modelJoin.matches(queryJoin)) { - matchJoin = modelJoin; - break; + for (OLAPContext ctx : contexts) { + for (JoinDesc queryJoin : ctx.joins) { + String fkTable = queryJoin.getForeignKeyColumns()[0].getTable(); + List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable); + if (modelJoins == null) + return null; + + JoinDesc matchJoin = null; + for (JoinDesc modelJoin : modelJoins) { + if (modelJoin.matches(queryJoin)) { + matchJoin = modelJoin; + break; + } } + if (matchJoin == null) + return null; + + matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result); + matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result); } - if (matchJoin == null) - return null; - matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result); - matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result); + OLAPTableScan firstTable = ctx.firstTableScan; + String firstTableAlias = firstTable.getAlias(); + if (result.containsKey(firstTableAlias) == false) { + TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName()); + if (tableRef == null) + return null; + matchesAdd(firstTableAlias, tableRef.getAlias(), result); + } } + return result; } @@ -99,10 +114,12 @@ public class ModelChooser { Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias)); } - private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) { - KylinConfig kylinConfig = context.olapSchema.getConfig(); - String projectName = context.olapSchema.getProjectName(); - String factTableName = context.firstTableScan.getOlapTable().getTableName(); + private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) { + // the first context, which is the top most context, contains all columns from all contexts + OLAPContext first = contexts.get(0); + KylinConfig kylinConfig = first.olapSchema.getConfig(); + String projectName = first.olapSchema.getProjectName(); + String factTableName = first.firstTableScan.getOlapTable().getTableName(); Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap(); @@ -110,7 +127,7 @@ public class ModelChooser { for (IRealization real : realizations) { if (real.isReady() == false) continue; - if (real.getAllColumns().containsAll(context.allColumns) == false) + if (containsAll(real.getAllColumnDescs(), first.allColumns) == false) continue; if (RemoveBlackoutRealizationsRule.accept(real) == false) continue; @@ -143,20 +160,28 @@ public class ModelChooser { return result; } + private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) { + for (TblColRef col : allColumns) { + if (allColumnDescs.contains(col.getColumnDesc()) == false) + return false; + } + return true; + } + private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) { for (OLAPTableScan tableScan : context.allTableScans) { tableScan.fixColumnRowTypeWithModel(model, aliasMap); } } - + private static class RealizationCost implements Comparable<RealizationCost> { final public int priority; final public int cost; - + public RealizationCost(IRealization real) { // ref Candidate.PRIORITIES this.priority = Candidate.PRIORITIES.get(real.getType()); - + // ref CubeInstance.getCost() int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE; for (LookupDesc lookup : real.getDataModelDesc().getLookups()) { @@ -165,7 +190,7 @@ public class ModelChooser { } this.cost = c; } - + @Override public int compareTo(RealizationCost o) { int comp = this.priority - o.priority;
