Repository: kylin Updated Branches: refs/heads/master 00c8f31e6 -> 3a36c66a1
KYLIN-2348 allow multiple models serving one query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a36c66a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a36c66a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a36c66a Branch: refs/heads/master Commit: 3a36c66a1c2bbb372b0a3f3243f181332d7ed882 Parents: 00c8f31 Author: Li Yang <liy...@apache.org> Authored: Tue Jan 10 11:32:58 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Tue Jan 10 11:44:17 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/Dictionary.java | 2 - .../apache/kylin/query/ITKylinQueryTest.java | 16 +++++-- .../org/apache/kylin/query/KylinTestBase.java | 1 + .../src/test/resources/query/sql/query101.sql | 18 +++++++ .../src/test/resources/query/sql/query102.sql | 18 +++++++ .../src/test/resources/query/sql/query103.sql | 18 +++++++ .../resources/query/sql_multi_model/query01.sql | 49 ++++++++++++++++++++ .../relnode/OLAPToEnumerableConverter.java | 18 +++---- .../kylin/query/routing/ModelChooser.java | 43 +++++++++++++---- .../apache/kylin/query/schema/OLAPTable.java | 8 ++-- 10 files changed, 161 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java index 03996a7..1e172bc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java @@ -254,6 +254,4 @@ abstract public class Dictionary<T> implements Serializable { */ public abstract void readFields(DataInput in) throws IOException; - - } http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 840a03f..aaf0925 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -47,7 +47,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; -@Ignore("KylinQueryTest is contained by ITCombinationTest") +//@Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { @Rule @@ -149,7 +149,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { - String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/temp/sample.sql"; + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_multi_model/query01.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { @@ -270,7 +270,17 @@ public class ITKylinQueryTest extends KylinTestBase { this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_intersect_count"); } } - + + + @Test + public void testMultiModelQuery() throws Exception { + if ("left".equalsIgnoreCase(joinType)) { + joinType = "default"; + execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_multi_model", null, true); + joinType = "left"; + } + } + @Test public void testDimDistinctCountQuery() throws Exception { execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true); http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 4f1f591..ae69f09 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -335,6 +335,7 @@ public class KylinTestBase { printInfo("Query Result from H2 - " + queryName); H2Connection h2Conn = new H2Connection(h2Connection, null); h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); + h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false); executeQuery(h2Conn, queryName, sql, needSort); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query101.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query101.sql b/kylin-it/src/test/resources/query/sql/query101.sql index fb42bca..765d26d 100644 --- a/kylin-it/src/test/resources/query/sql/query101.sql +++ b/kylin-it/src/test/resources/query/sql/query101.sql @@ -1,3 +1,21 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + select meta_categ_name, count(1) as cnt, sum(price) as GMV http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query102.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query102.sql b/kylin-it/src/test/resources/query/sql/query102.sql index bd1e15e..6073c98 100644 --- a/kylin-it/src/test/resources/query/sql/query102.sql +++ b/kylin-it/src/test/resources/query/sql/query102.sql @@ -1,3 +1,21 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + select meta_categ_name, count(1) as cnt, sum(price) as GMV http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query103.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query103.sql b/kylin-it/src/test/resources/query/sql/query103.sql index c5f9bf9..6c8ce9e 100644 --- a/kylin-it/src/test/resources/query/sql/query103.sql +++ b/kylin-it/src/test/resources/query/sql/query103.sql @@ -1,3 +1,21 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + select meta_categ_name, count(1) as cnt, sum(price) as GMV http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql_multi_model/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_multi_model/query01.sql b/kylin-it/src/test/resources/query/sql_multi_model/query01.sql new file mode 100644 index 0000000..cb0e2f2 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_multi_model/query01.sql @@ -0,0 +1,49 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select + l.cal_dt + , sum(left_join_gvm) as left_join_sum + , sum(inner_join_gvm) as inner_join_sum +from + ( + select test_kylin_fact.cal_dt, sum(price) as left_join_gvm + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + + group by test_kylin_fact.cal_dt + ) l + inner join + ( + select test_kylin_fact.cal_dt, sum(price) as inner_join_gvm + + from test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + + group by test_kylin_fact.cal_dt + ) i + on l.cal_dt = i.cal_dt +group by + l.cal_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java index 8f01a74..0468c91 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java @@ -18,6 +18,7 @@ package org.apache.kylin.query.relnode; +import java.util.IdentityHashMap; import java.util.List; import java.util.Set; @@ -79,19 +80,11 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab // identify model List<OLAPContext> contexts = listContextsHavingScan(); - Set<IRealization> candidates = ModelChooser.selectModel(contexts); + IdentityHashMap<OLAPContext, Set<IRealization>> candidates = ModelChooser.selectModel(contexts); // identify realization for each context - for (OLAPContext context : OLAPContext.getThreadLocalContexts()) { - - // Context has no table scan is created by OLAPJoinRel which looks like - // (sub-query) as A join (sub-query) as B - // No realization needed for such context. - if (context.firstTableScan == null) { - continue; - } - - IRealization realization = QueryRouter.selectRealization(context, candidates); + for (OLAPContext context : contexts) { + IRealization realization = QueryRouter.selectRealization(context, candidates.get(context)); context.realization = realization; doAccessControl(context); } @@ -115,6 +108,9 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab } private List<OLAPContext> listContextsHavingScan() { + // Context has no table scan is created by OLAPJoinRel which looks like + // (sub-query) as A join (sub-query) as B + // No realization needed for such context. int size = OLAPContext.getThreadLocalContexts().size(); List<OLAPContext> result = Lists.newArrayListWithCapacity(size); for (int i = 0; i < size; i++) { http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java index ccb2900..2517dc5 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java +++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java @@ -19,6 +19,7 @@ package org.apache.kylin.query.routing; import java.util.Comparator; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -39,14 +40,39 @@ import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.relnode.OLAPTableScan; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; public class ModelChooser { - // select a model that satisfies all the contexts - public static Set<IRealization> selectModel(List<OLAPContext> contexts) { + // select models for given contexts, return realization candidates for each context + public static IdentityHashMap<OLAPContext, Set<IRealization>> selectModel(List<OLAPContext> contexts) { + + IdentityHashMap<OLAPContext, Set<IRealization>> candidates = new IdentityHashMap<>(); + + // attempt one model for all contexts + Set<IRealization> reals = attemptSelectModel(contexts); + if (reals != null) { + for (OLAPContext ctx : contexts) { + candidates.put(ctx, reals); + } + return candidates; + } + + // try different model for different context + for (OLAPContext ctx : contexts) { + reals = attemptSelectModel(ImmutableList.of(ctx)); + if (reals == null) + throw new NoRealizationFoundException("No model found for" + toErrorMsg(ctx)); + + candidates.put(ctx, reals); + } + return candidates; + } + + private static Set<IRealization> attemptSelectModel(List<OLAPContext> contexts) { Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(contexts); for (DataModelDesc model : modelMap.keySet()) { @@ -57,17 +83,14 @@ public class ModelChooser { return modelMap.get(model); } } - - throw new NoRealizationFoundException("No model found for" + toErrorMsg(contexts)); + return null; } - private static String toErrorMsg(List<OLAPContext> contexts) { + private static String toErrorMsg(OLAPContext ctx) { StringBuilder buf = new StringBuilder(); - for (OLAPContext ctx : contexts) { - buf.append(", ").append(ctx.firstTableScan); - for (JoinDesc join : ctx.joins) - buf.append(", ").append(join); - } + buf.append(ctx.firstTableScan); + for (JoinDesc join : ctx.joins) + buf.append(", ").append(join); return buf.toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java index 85996f4..fe6a806 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java @@ -186,14 +186,14 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab //to avoid overflow we upgrade x's type to long //this includes checking two parts: //1. sum measures in cubes: - HashSet<ColumnDesc> updateColumns = Sets.newHashSet(); + HashSet<ColumnDesc> upgradeCols = Sets.newHashSet(); for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) { if (m.getFunction().isSum()) { FunctionDesc func = m.getFunction(); if (func.getReturnDataType() != func.getRewriteFieldType() && // func.getReturnDataType().isBigInt() && // func.getRewriteFieldType().isIntegerFamily()) { - updateColumns.add(func.getParameter().getColRefs().get(0).getColumnDesc()); + upgradeCols.add(func.getParameter().getColRefs().get(0).getColumnDesc()); } } } @@ -204,12 +204,12 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab for (String metricColumn : model.getMetrics()) { TblColRef col = model.findColumn(metricColumn); if (col.getTable().equals(sourceTable.getIdentity()) && col.getType().isIntegerFamily() && !col.getType().isBigInt()) - updateColumns.add(col.getColumnDesc()); + upgradeCols.add(col.getColumnDesc()); } } } - for (ColumnDesc upgrade : updateColumns) { + for (ColumnDesc upgrade : upgradeCols) { int index = tableColumns.indexOf(upgrade); if (index < 0) { throw new IllegalStateException("Metric column " + upgrade + " is not found in the the project's columns");
