Repository: kylin Updated Branches: refs/heads/KYLIN-1971 ea01024d9 -> 69749e562
tmp Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69749e56 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69749e56 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69749e56 Branch: refs/heads/KYLIN-1971 Commit: 69749e5622983455aa65bb7f440d7d5321fd727a Parents: ea01024 Author: Yang Li <liy...@apache.org> Authored: Tue Oct 25 08:19:15 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Tue Oct 25 08:19:15 2016 +0800 ---------------------------------------------------------------------- .../kylin/query/relnode/OLAPTableScan.java | 16 ++-- .../relnode/OLAPToEnumerableConverter.java | 5 +- .../kylin/query/routing/ModelChooser.java | 93 ++++++++++++++++++-- .../apache/kylin/query/routing/QueryRouter.java | 6 +- 4 files changed, 101 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/69749e56/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index aa70dbc..8b5ad78 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -84,6 +84,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { private final OLAPTable olapTable; private final String tableName; private final int[] fields; + private String alias; private ColumnRowType columnRowType; private OLAPContext context; @@ -222,9 +223,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { } } + public String getAlias() { + return alias; + } + private ColumnRowType buildColumnRowType() { - String tmpAlias = Integer.toHexString(System.identityHashCode(this)); - TableRef tableRef = TblColRef.tableForUnknownModel(tmpAlias, olapTable.getSourceTable()); + this.alias = Integer.toHexString(System.identityHashCode(this)); + TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, olapTable.getSourceTable()); List<TblColRef> columns = new ArrayList<TblColRef>(); for (ColumnDesc sourceColumn : olapTable.getExposedColumns()) { @@ -239,11 +244,12 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { return tableRef.makeFakeColumn(name); } - public void fixColumnRowTypeWithModel(DataModelDesc model) { - TableRef tableRef = model.findFirstTable(olapTable.getTableName()); + public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) { + String newAlias = aliasMap.get(this.alias); for (TblColRef col : columnRowType.getAllColumns()) { - TblColRef.fixUnknownModel(model, tableRef.getAlias(), col); + TblColRef.fixUnknownModel(model, newAlias, col); } + this.alias = newAlias; } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/69749e56/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java index f34893a..24fc430 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java @@ -45,7 +45,6 @@ import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.LogicalTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.routing.ModelChooser; @@ -90,8 +89,8 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab continue; } - ModelChooser.chooseModel(context); - IRealization realization = QueryRouter.selectRealization(context); + Set<IRealization> candidates = ModelChooser.selectModel(context); + IRealization realization = QueryRouter.selectRealization(context, candidates); context.realization = realization; doAccessControl(context); http://git-wip-us.apache.org/repos/asf/kylin/blob/69749e56/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java index 972efd0..9b1b007 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java +++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java @@ -18,21 +18,102 @@ package org.apache.kylin.query.routing; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.TableRef; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.relnode.OLAPTableScan; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + public class ModelChooser { - public static void chooseModel(OLAPContext context) { - DataModelDesc model = null; + public static Set<IRealization> selectModel(OLAPContext context) { + Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context); + OLAPTableScan firstTable = context.firstTableScan; + List<JoinDesc> joins = context.joins; + + for (DataModelDesc model : modelMap.keySet()) { + Map<String, String> aliasMap = matches(model, firstTable, joins); + if (aliasMap != null) { + fixModel(context, model, aliasMap); + return modelMap.get(model); + } + } + + throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins); + } + + private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) { + Map<String, String> result = Maps.newHashMap(); + + // no join special case + if (joins.isEmpty()) { + TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName()); + if (tableRef == null) + return null; + result.put(firstTable.getAlias(), tableRef.getAlias()); + return result; + } - fixModel(context, model); + // the greedy match is not perfect but works for the moment + + return null; } - - private static void fixModel(OLAPContext context, DataModelDesc model) { + + private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) { + KylinConfig kylinConfig = context.olapSchema.getConfig(); + String projectName = context.olapSchema.getProjectName(); + String factTableName = context.firstTableScan.getOlapTable().getTableName(); + Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); + + final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap(); + final Map<DataModelDesc, MutableLong> endTimes = Maps.newHashMap(); + for (IRealization real : realizations) { + if (real.isReady() == false) + continue; + + DataModelDesc m = real.getDataModelDesc(); + Set<IRealization> set = models.get(m); + if (set == null) { + set = Sets.newHashSet(); + set.add(real); + models.put(m, set); + endTimes.put(m, new MutableLong(real.getDateRangeEnd())); + } else { + set.add(real); + MutableLong endTime = endTimes.get(m); + endTime.setValue(Math.max(real.getDateRangeEnd(), endTime.longValue())); + } + } + + // order by end time desc + TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() { + @Override + public int compare(DataModelDesc o1, DataModelDesc o2) { + long comp = endTimes.get(o2).longValue() - endTimes.get(o1).longValue(); + return comp == 0 ? 0 : (comp < 0 ? -1 : 1); + } + }); + result.putAll(models); + + return result; + } + + private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) { for (OLAPTableScan tableScan : context.allTableScans) { - tableScan.fixColumnRowTypeWithModel(model); + tableScan.fixColumnRowTypeWithModel(model, aliasMap); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/69749e56/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java index 69ebfa6..2975cf7 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java +++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java @@ -23,7 +23,6 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.CapabilityResult; import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.CapabilityResult.DimensionAsMeasure; @@ -42,13 +41,10 @@ public class QueryRouter { private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class); - public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException { + public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) throws NoRealizationFoundException { - ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig()); - logger.info("The project manager's reference is " + prjMgr); String factTableName = olapContext.firstTableScan.getTableName(); String projectName = olapContext.olapSchema.getProjectName(); - Set<IRealization> realizations = prjMgr.getRealizationsByTable(projectName, factTableName); SQLDigest sqlDigest = olapContext.getSQLDigest(); List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
