KYLIN-1971 ModelChooser, match Model against query joins
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cdd2252 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cdd2252 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cdd2252 Branch: refs/heads/KYLIN-1971 Commit: 5cdd22523a68faa4879092d4b2510ea461609f6e Parents: 151ec3b Author: Yang Li <liy...@apache.org> Authored: Tue Oct 25 06:16:15 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Wed Oct 26 11:06:42 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeInstance.java | 13 +- .../apache/kylin/metadata/model/ColumnDesc.java | 8 +- .../kylin/metadata/model/DataModelDesc.java | 13 +- .../kylin/metadata/model/FunctionDesc.java | 18 -- .../apache/kylin/metadata/model/JoinDesc.java | 44 ++++- .../apache/kylin/metadata/model/TableRef.java | 4 +- .../apache/kylin/metadata/model/TblColRef.java | 21 ++- .../kylin/metadata/project/ProjectL2Cache.java | 2 +- .../metadata/realization/IRealization.java | 123 ++++++------- .../kylin/storage/hybrid/HybridInstance.java | 7 +- .../apache/kylin/query/ITCombinationTest.java | 9 +- .../kylin/query/relnode/OLAPTableScan.java | 16 +- .../relnode/OLAPToEnumerableConverter.java | 13 +- .../kylin/query/routing/ModelChooser.java | 178 +++++++++++++++++++ .../apache/kylin/query/routing/QueryRouter.java | 6 +- .../rules/RemoveBlackoutRealizationsRule.java | 18 +- 16 files changed, 351 insertions(+), 142 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 720690d..2ccdde7 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -21,6 +21,7 @@ package org.apache.kylin.cube; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; @@ -52,9 +53,9 @@ import com.google.common.collect.Lists; @SuppressWarnings("serial") @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class CubeInstance extends RootPersistentEntity implements IRealization, IBuildable { - private static final int COST_WEIGHT_MEASURE = 1; - private static final int COST_WEIGHT_DIMENSION = 10; - private static final int COST_WEIGHT_INNER_JOIN = 100; + public static final int COST_WEIGHT_MEASURE = 1; + public static final int COST_WEIGHT_DIMENSION = 10; + public static final int COST_WEIGHT_INNER_JOIN = 100; public static CubeInstance create(String cubeName, CubeDesc cubeDesc) { CubeInstance cubeInstance = new CubeInstance(); @@ -374,7 +375,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) { // more tables, more cost - if ("inner".equals(lookupDesc.getJoin().getType())) { + if (lookupDesc.getJoin().isInnerJoin()) { // inner join cost is bigger than left join, as it will filter some records calculatedCost += COST_WEIGHT_INNER_JOIN; } @@ -389,8 +390,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } @Override - public List<TblColRef> getAllColumns() { - return Lists.newArrayList(getDescriptor().listAllColumns()); + public Set<TblColRef> getAllColumns() { + return getDescriptor().listAllColumns(); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java index 772297f..3bf0de9 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java @@ -20,12 +20,11 @@ package org.apache.kylin.metadata.model; import java.io.Serializable; -import com.fasterxml.jackson.annotation.JsonInclude; -import org.apache.commons.lang.StringUtils; import org.apache.kylin.metadata.datatype.DataType; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; /** @@ -154,11 +153,6 @@ public class ColumnDesc implements Serializable { } } - public boolean isSameAs(String tableName, String columnName) { - return StringUtils.equalsIgnoreCase(table.getIdentity(), tableName) && // - StringUtils.equalsIgnoreCase(name, columnName); - } - @Override public int hashCode() { final int prime = 31; http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index a7734f1..7c39a25 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -86,6 +86,7 @@ public class DataModelDesc extends RootPersistentEntity { private List<TableRef> lookupTableRefs = Lists.newArrayList(); private Map<String, TableRef> aliasMap = Maps.newHashMap(); // a table has exactly one alias private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // a table maybe referenced by multiple names + private Map<String, List<JoinDesc>> joinsMap = Maps.newHashMap(); /** * Error messages during resolving json metadata @@ -274,6 +275,7 @@ public class DataModelDesc extends RootPersistentEntity { lookupTableRefs.clear(); aliasMap.clear(); tableNameMap.clear(); + joinsMap.clear(); initTableAlias(tables); initJoinColumns(); @@ -378,10 +380,19 @@ public class DataModelDesc extends RootPersistentEntity { } } + List<JoinDesc> list = joinsMap.get(factTableRef.getTableIdentity()); + if (list == null) + joinsMap.put(factTableRef.getTableIdentity(), list = Lists.newArrayListWithCapacity(4)); + list.add(join); } } + + public Map<String, List<JoinDesc>> getJoinsMap() { + return joinsMap; + } - /** * Add error info and thrown exception out + /** + * Add error info and thrown exception out * * @param message */ http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index a2c5756..e7d5186 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -19,7 +19,6 @@ package org.apache.kylin.metadata.model; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Set; @@ -238,23 +237,6 @@ public class FunctionDesc { this.returnDataType = DataType.getType(returnType); } - public TblColRef selectTblColRef(Collection<TblColRef> metricColumns, String factTableName) { - if (this.isCount()) - return null; // count is not about any column but the whole row - - ParameterDesc parameter = this.getParameter(); - if (parameter == null) - return null; - - String columnName = parameter.getValue(); - for (TblColRef col : metricColumns) { - if (col.isSameAs(factTableName, columnName)) { - return col; - } - } - return null; - } - public HashMap<String, String> getConfiguration() { return configuration; } http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java index 458367d..04fbf62 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java @@ -50,10 +50,18 @@ public class JoinDesc { foreignKeyColumns = tt; } + public boolean isInnerJoin() { + return "INNER".equalsIgnoreCase(type); + } + + public boolean isLeftJoin() { + return "LEFT".equalsIgnoreCase(type); + } + public String getType() { return type; } - + public void setType(String type) { this.type = type; } @@ -110,9 +118,9 @@ public class JoinDesc { return false; JoinDesc other = (JoinDesc) obj; - if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns)) + if (!this.colRefsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns)) return false; - if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns)) + if (!this.colRefsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns)) return false; if (!this.type.equalsIgnoreCase(other.getType())) @@ -120,13 +128,41 @@ public class JoinDesc { return true; } - private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) { + private boolean colRefsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) { if (a.length != b.length) return false; return Arrays.asList(a).containsAll(Arrays.asList(b)); } + // equals() without alias + public boolean matches(JoinDesc other) { + if (!this.type.equalsIgnoreCase(other.getType())) + return false; + if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns)) + return false; + if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns)) + return false; + + return true; + } + + private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) { + if (a.length != b.length) + return false; + + int match = 0; + for (int i = 0; i < a.length; i++) { + for (int j = 0; j < b.length; j++) { + if (a[i].equals(b[j])) { + match++; + break; + } + } + } + return match == a.length; + } + @Override public String toString() { return "JoinDesc [type=" + type + ", primary_key=" + Arrays.toString(primaryKey) + ", foreign_key=" + Arrays.toString(foreignKey) + "]"; http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java index 186ff35..254ce18 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java @@ -86,9 +86,9 @@ public class TableRef { TableRef t = (TableRef) o; - if (!model.equals(t.model)) + if ((model == null ? t.model == null : model.getName().equals(t.model.getName())) == false) return false; - if (!alias.equals(t.alias)) + if ((alias == null ? t.alias == null : alias.equals(t.alias)) == false) return false; if (!table.getIdentity().equals(t.table.getIdentity())) return false; http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index 69569dd..8841ee5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -64,6 +64,9 @@ public class TblColRef implements Serializable { } private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc(); + static { + UNKNOWN_MODEL.setName("UNKNOWN_MODEL"); + } public static TableRef tableForUnknownModel(String tempTableAlias, TableDesc table) { return new TableRef(UNKNOWN_MODEL, tempTableAlias, table); @@ -119,6 +122,10 @@ public class TblColRef implements Serializable { return table; } + public String getTableAlias() { + return table.getAlias(); + } + public String getTable() { if (column.getTable() == null) { return null; @@ -152,17 +159,6 @@ public class TblColRef implements Serializable { return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype()); } - /** - * - * @param tableName full name : db.table - * @param columnName columnname - * @return - */ - public boolean isSameAs(String tableName, String columnName) { - return column.isSameAs(tableName, columnName); - } - - @Override public int hashCode() { // NOTE: tableRef MUST NOT participate in hashCode(). // Because fixUnknownModel() can change tableRef while TblColRef is held as set/map keys. @@ -187,6 +183,8 @@ public class TblColRef implements Serializable { return false; if (!StringUtils.equals(column.getName(), other.column.getName())) return false; +// if ((table == null ? other.table == null : table.equals(other.table)) == false) +// return false; return true; } @@ -201,4 +199,5 @@ public class TblColRef implements Serializable { return alias + ":" + tableIdentity + "." + column.getName(); } } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java index 1f883bd..15fa8ce 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java @@ -233,7 +233,7 @@ class ProjectL2Cache { MetadataManager metaMgr = mgr.getMetadataManager(); - List<TblColRef> allColumns = realization.getAllColumns(); + Set<TblColRef> allColumns = realization.getAllColumns(); if (allColumns == null || allColumns.isEmpty()) { logger.error("Realization '" + realization.getCanonicalName() + "' does not report any columns"); return false; http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index 040cdc5..343ec99 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -1,61 +1,62 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.metadata.realization; - -import java.util.List; - -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.IStorageAware; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.TblColRef; - -public interface IRealization extends IStorageAware { - - /** - * Given the features of a query, check how capable the realization is to answer the query. - */ - public CapabilityResult isCapable(SQLDigest digest); - - /** - * Get whether this specific realization is a cube or InvertedIndex - */ - public RealizationType getType(); - - public DataModelDesc getDataModelDesc(); - - public String getFactTable(); - - public List<TblColRef> getAllColumns(); - - public List<TblColRef> getAllDimensions(); - - public List<MeasureDesc> getMeasures(); - - public boolean isReady(); - - public String getName(); - - public String getCanonicalName(); - - public long getDateRangeStart(); - - public long getDateRangeEnd(); - - public boolean supportsLimitPushDown(); -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.metadata.realization; + +import java.util.List; +import java.util.Set; + +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.IStorageAware; +import org.apache.kylin.metadata.model.MeasureDesc; +import org.apache.kylin.metadata.model.TblColRef; + +public interface IRealization extends IStorageAware { + + /** + * Given the features of a query, check how capable the realization is to answer the query. + */ + public CapabilityResult isCapable(SQLDigest digest); + + /** + * Get whether this specific realization is a cube or InvertedIndex + */ + public RealizationType getType(); + + public DataModelDesc getDataModelDesc(); + + public String getFactTable(); + + public Set<TblColRef> getAllColumns(); + + public List<TblColRef> getAllDimensions(); + + public List<MeasureDesc> getMeasures(); + + public boolean isReady(); + + public String getName(); + + public String getCanonicalName(); + + public long getDateRangeStart(); + + public long getDateRangeEnd(); + + public boolean supportsLimitPushDown(); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 9b3a0fc..57d14d5 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; @@ -66,7 +67,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization private volatile IRealization[] realizations = null; private List<TblColRef> allDimensions = null; - private List<TblColRef> allColumns = null; + private Set<TblColRef> allColumns = null; private List<MeasureDesc> allMeasures = null; private long dateRangeStart; private long dateRangeEnd; @@ -136,7 +137,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization } allDimensions = Lists.newArrayList(dimensions); - allColumns = Lists.newArrayList(columns); + allColumns = columns; allMeasures = Lists.newArrayList(measures); Collections.sort(realizationList, new Comparator<IRealization>() { @@ -203,7 +204,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization } @Override - public List<TblColRef> getAllColumns() { + public Set<TblColRef> getAllColumns() { init(); return allColumns; } http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index f4667af..496ac4e 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -65,7 +65,14 @@ public class ITCombinationTest extends ITKylinQueryTest { */ @Parameterized.Parameters public static Collection<Object[]> configs() { - return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on", "v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on", "v2", true } }); + return Arrays.asList(new Object[][] { // + { "inner", "on", "v2", true }, // + { "left", "on", "v1", true }, // + { "left", "on", "v2", true }, // + //{ "inner", "on", "v2", false }, // exclude view to simply model/cube selection + //{ "left", "on", "v1", false }, // exclude view to simply model/cube selection + //{ "left", "on", "v2", false }, // exclude view to simply model/cube selection + }); } public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index aa70dbc..8b5ad78 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -84,6 +84,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { private final OLAPTable olapTable; private final String tableName; private final int[] fields; + private String alias; private ColumnRowType columnRowType; private OLAPContext context; @@ -222,9 +223,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { } } + public String getAlias() { + return alias; + } + private ColumnRowType buildColumnRowType() { - String tmpAlias = Integer.toHexString(System.identityHashCode(this)); - TableRef tableRef = TblColRef.tableForUnknownModel(tmpAlias, olapTable.getSourceTable()); + this.alias = Integer.toHexString(System.identityHashCode(this)); + TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, olapTable.getSourceTable()); List<TblColRef> columns = new ArrayList<TblColRef>(); for (ColumnDesc sourceColumn : olapTable.getExposedColumns()) { @@ -239,11 +244,12 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { return tableRef.makeFakeColumn(name); } - public void fixColumnRowTypeWithModel(DataModelDesc model) { - TableRef tableRef = model.findFirstTable(olapTable.getTableName()); + public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) { + String newAlias = aliasMap.get(this.alias); for (TblColRef col : columnRowType.getAllColumns()) { - TblColRef.fixUnknownModel(model, tableRef.getAlias(), col); + TblColRef.fixUnknownModel(model, newAlias, col); } + this.alias = newAlias; } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java index aa01086..24fc430 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java @@ -45,9 +45,9 @@ import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.LogicalTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.query.routing.ModelChooser; import org.apache.kylin.query.routing.NoRealizationFoundException; import org.apache.kylin.query.routing.QueryRouter; import org.apache.kylin.query.schema.OLAPTable; @@ -89,10 +89,10 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab continue; } - IRealization realization = QueryRouter.selectRealization(context); + Set<IRealization> candidates = ModelChooser.selectModel(context); + IRealization realization = QueryRouter.selectRealization(context, candidates); context.realization = realization; - fixModel(context); doAccessControl(context); } } catch (NoRealizationFoundException e) { @@ -123,13 +123,6 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab return impl.visitChild(this, 0, inputAsEnum, pref); } - private void fixModel(OLAPContext context) { - DataModelDesc model = context.realization.getDataModelDesc(); - for (OLAPTableScan tableScan : context.allTableScans) { - tableScan.fixColumnRowTypeWithModel(model); - } - } - private void doAccessControl(OLAPContext context) { String controllerCls = KylinConfig.getInstanceFromEnv().getQueryAccessController(); if (null != controllerCls && !controllerCls.isEmpty()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java new file mode 100644 index 0000000..4667f4a --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.routing; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.JoinDesc; +import org.apache.kylin.metadata.model.LookupDesc; +import org.apache.kylin.metadata.model.TableRef; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.query.relnode.OLAPContext; +import org.apache.kylin.query.relnode.OLAPTableScan; +import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +public class ModelChooser { + + public static Set<IRealization> selectModel(OLAPContext context) { + Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context); + OLAPTableScan firstTable = context.firstTableScan; + List<JoinDesc> joins = context.joins; + + for (DataModelDesc model : modelMap.keySet()) { + Map<String, String> aliasMap = matches(model, firstTable, joins); + if (aliasMap != null) { + fixModel(context, model, aliasMap); + return modelMap.get(model); + } + } + + throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins); + } + + private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) { + Map<String, String> result = Maps.newHashMap(); + + // no join special case + if (joins.isEmpty()) { + TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName()); + if (tableRef == null) + return null; + result.put(firstTable.getAlias(), tableRef.getAlias()); + return result; + } + + // the greedy match is not perfect but works for the moment + Map<String, List<JoinDesc>> modelJoinsMap = model.getJoinsMap(); + for (JoinDesc queryJoin : joins) { + String fkTable = queryJoin.getForeignKeyColumns()[0].getTable(); + List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable); + if (modelJoins == null) + return null; + + JoinDesc matchJoin = null; + for (JoinDesc modelJoin : modelJoins) { + if (modelJoin.matches(queryJoin)) { + matchJoin = modelJoin; + break; + } + } + if (matchJoin == null) + return null; + + matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result); + matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result); + } + return result; + } + + private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) { + String existingTarget = result.put(origAlias, targetAlias); + Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias)); + } + + private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) { + KylinConfig kylinConfig = context.olapSchema.getConfig(); + String projectName = context.olapSchema.getProjectName(); + String factTableName = context.firstTableScan.getOlapTable().getTableName(); + Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); + + final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap(); + final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap(); + for (IRealization real : realizations) { + if (real.isReady() == false) + continue; + if (real.getAllColumns().containsAll(context.allColumns) == false) + continue; + if (RemoveBlackoutRealizationsRule.accept(real) == false) + continue; + + RealizationCost cost = new RealizationCost(real); + DataModelDesc m = real.getDataModelDesc(); + Set<IRealization> set = models.get(m); + if (set == null) { + set = Sets.newHashSet(); + set.add(real); + models.put(m, set); + costs.put(m, cost); + } else { + set.add(real); + RealizationCost curCost = costs.get(m); + if (cost.compareTo(curCost) < 0) + costs.put(m, cost); + } + } + + // order model by cheapest realization cost + TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() { + @Override + public int compare(DataModelDesc o1, DataModelDesc o2) { + return costs.get(o1).compareTo(costs.get(o2)); + } + }); + result.putAll(models); + + return result; + } + + private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) { + for (OLAPTableScan tableScan : context.allTableScans) { + tableScan.fixColumnRowTypeWithModel(model, aliasMap); + } + } + + private static class RealizationCost implements Comparable<RealizationCost> { + final public int priority; + final public int cost; + + public RealizationCost(IRealization real) { + // ref Candidate.PRIORITIES + this.priority = Candidate.PRIORITIES.get(real.getType()); + + // ref CubeInstance.getCost() + int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE; + for (LookupDesc lookup : real.getDataModelDesc().getLookups()) { + if (lookup.getJoin().isInnerJoin()) + c += CubeInstance.COST_WEIGHT_INNER_JOIN; + } + this.cost = c; + } + + @Override + public int compareTo(RealizationCost o) { + int comp = this.priority - o.priority; + if (comp != 0) + return comp; + else + return this.cost - o.cost; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java index 69ebfa6..2975cf7 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java +++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java @@ -23,7 +23,6 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.CapabilityResult; import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence; import org.apache.kylin.metadata.realization.CapabilityResult.DimensionAsMeasure; @@ -42,13 +41,10 @@ public class QueryRouter { private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class); - public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException { + public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) throws NoRealizationFoundException { - ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig()); - logger.info("The project manager's reference is " + prjMgr); String factTableName = olapContext.firstTableScan.getTableName(); String projectName = olapContext.olapSchema.getProjectName(); - Set<IRealization> realizations = prjMgr.getRealizationsByTable(projectName, factTableName); SQLDigest sqlDigest = olapContext.getSQLDigest(); List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size()); http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java index f299d17..f309757 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java +++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.routing.Candidate; import org.apache.kylin.query.routing.RoutingRule; @@ -34,19 +35,22 @@ public class RemoveBlackoutRealizationsRule extends RoutingRule { public static Set<String> blackList = Sets.newHashSet(); public static Set<String> whiteList = Sets.newHashSet(); + public static boolean accept(IRealization real) { + if (blackList.contains(real.getCanonicalName())) + return false; + if (!whiteList.isEmpty() && !whiteList.contains(real.getCanonicalName())) + return false; + + return true; + } + @Override public void apply(List<Candidate> candidates) { for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) { Candidate candidate = iterator.next(); - if (blackList.contains(candidate.getRealization().getCanonicalName())) { - iterator.remove(); - continue; - } - - if (!whiteList.isEmpty() && !whiteList.contains(candidate.getRealization().getCanonicalName())) { + if (!accept(candidate.getRealization())) { iterator.remove(); - continue; } } }
