KYLIN-2599 more complete fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f277b2d5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f277b2d5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f277b2d5 Branch: refs/heads/master Commit: f277b2d586d4b306d15a2fc7de83b1dfb38d628e Parents: c3d8183 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed May 10 13:52:14 2017 +0800 Committer: liyang-gmt8 <liy...@apache.org> Committed: Wed May 10 14:16:18 2017 +0800 ---------------------------------------------------------------------- .../calcite/sql2rel/SqlToRelConverter.java | 43 ++++++++++++++------ .../apache/kylin/query/ITKylinQueryTest.java | 1 + .../resources/query/sql_verifyCount/query09.sql | 34 ++++++++++++++++ .../query/sql_verifyCount/query09.sql.expected | 2 + .../resources/query/sql_verifyCount/query10.sql | 34 ++++++++++++++++ .../query/sql_verifyCount/query10.sql.expected | 2 + .../resources/query/sql_verifyCount/query11.sql | 36 ++++++++++++++++ .../query/sql_verifyCount/query11.sql.expected | 2 + 8 files changed, 142 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java ---------------------------------------------------------------------- diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index a520aa6..8085a70 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -29,6 +29,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.prepare.Prepare; import org.apache.calcite.prepare.RelOptTableImpl; import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollationImpl; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; @@ -613,15 +614,6 @@ public class SqlToRelConverter { return root; } - RelNode tableScanOrJoin = null; - RelNode rootProjInput = rootPrj.getInput(0) instanceof LogicalFilter ? rootPrj.getInput(0).getInput(0): rootPrj.getInput(0); - if(rootProjInput instanceof TableScan || rootProjInput instanceof LogicalJoin) { - tableScanOrJoin = rootProjInput; - } - if(tableScanOrJoin == null) { - return root; - } - // RelNode input = rootPrj.getInput(); // if (!(// @@ -637,25 +629,52 @@ public class SqlToRelConverter { List<String> inFields = inType.getFieldNames(); List<RexNode> projExp = new ArrayList<>(); List<Pair<Integer, String>> projFields = new ArrayList<>(); + Map<Integer,Integer> projFieldMapping = new HashMap<>(); RelDataTypeFactory.FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder(); RelDataTypeFactory.FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder(); + + boolean hiddenColumnExists = false; + for (int i = 0; i < root.validatedRowType.getFieldList().size(); i++) { + if (root.validatedRowType.getFieldNames().get(i).startsWith("_KY_")) + hiddenColumnExists = true; + } + if(!hiddenColumnExists) { + return root; + } + for (int i = 0; i < inFields.size(); i++) { if (!inFields.get(i).startsWith("_KY_")) { projExp.add(rootPrj.getProjects().get(i)); + projFieldMapping.put(i, projFields.size()); projFields.add(Pair.of(projFields.size(), inFields.get(i))); projTypeBuilder.add(inType.getFieldList().get(i)); - validTypeBuilder.add(root.validatedRowType.getFieldList().get(i)); + + if (i < root.validatedRowType.getFieldList().size()) //for cases like kylin-it/src/test/resources/query/sql_verifyCount/query10.sql + validTypeBuilder.add(root.validatedRowType.getFieldList().get(i)); } } RelDataType projRowType = getCluster().getTypeFactory().createStructType(projTypeBuilder); rootPrj = LogicalProject.create(input, projExp, projRowType); if (rootSort != null) { - rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch); + //for cases like kylin-it/src/test/resources/query/sql_verifyCount/query10.sql, original RelCollation is stale, need to fix its fieldIndex + RelCollation originalCollation = rootSort.collation; + RelCollation newCollation = null; + List<RelFieldCollation> fieldCollations = originalCollation.getFieldCollations(); + ImmutableList.Builder<RelFieldCollation> newFieldCollations = ImmutableList.builder(); + for (RelFieldCollation fieldCollation : fieldCollations) { + if(projFieldMapping.containsKey(fieldCollation.getFieldIndex())) { + newFieldCollations.add(fieldCollation.copy(projFieldMapping.get(fieldCollation.getFieldIndex()))); + } else { + newFieldCollations.add(fieldCollation); + } + } + newCollation = RelCollationImpl.of(newFieldCollations.build()); + rootSort = LogicalSort.create(rootPrj, newCollation, rootSort.offset, rootSort.fetch); } RelDataType validRowType = getCluster().getTypeFactory().createStructType(validTypeBuilder); - root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind, projFields, root.collation); + root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind, projFields, rootSort == null ? root.collation : rootSort.getCollation()); validator.setValidatedNodeType(query, validRowType); http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 629e861..55041e3 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -133,6 +133,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { + System.setProperty("log4j.configuration", "file:../build/conf/kylin-tools-log4j.properties"); String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_verifyCount/query03.sql"; http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql b/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql new file mode 100644 index 0000000..74a8355 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql @@ -0,0 +1,34 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + + + select lstg_format_name from test_kylin_fact order by case + + when 1=1 then + + cal_dt + + ELSE + + seller_id + + end + + + + http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql.expected new file mode 100644 index 0000000..dbab0e0 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query09.sql.expected @@ -0,0 +1,2 @@ +* +1 http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql new file mode 100644 index 0000000..9435f6d --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql @@ -0,0 +1,34 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + + + select * from test_kylin_fact order by case + + when 1=1 then + + cal_dt + + ELSE + + seller_id + + end desc + + + + http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected new file mode 100644 index 0000000..11636e7 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected @@ -0,0 +1,2 @@ +* +13 http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql new file mode 100644 index 0000000..84a6710 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql @@ -0,0 +1,36 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + + + select * from test_kylin_fact where case + + when cal_dt is null then + + date'2010-01-01' + + ELSE + + cal_dt + + end + + < date '2013-01-01' + limit 10 + + + http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected new file mode 100644 index 0000000..093e327 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected @@ -0,0 +1,2 @@ +10 +13
