KYLIN-2277 return correct columns for select *
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/654144f4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/654144f4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/654144f4 Branch: refs/heads/master-cdh5.7 Commit: 654144f47ff647bc4b7eac9793d2fa5f88dec372 Parents: e6e330a Author: Yang Li <liy...@apache.org> Authored: Tue Dec 13 23:45:55 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Wed Dec 14 21:18:21 2016 +0800 ---------------------------------------------------------------------- .../calcite/sql2rel/SqlToRelConverter.java | 62 +++++++++++++++++++- .../kylin/metadata/model/FunctionDesc.java | 4 +- .../apache/kylin/query/ITKylinQueryTest.java | 6 ++ .../org/apache/kylin/query/KylinTestBase.java | 21 +++++++ .../resources/query/sql_timeout/query01.sql | 2 +- 5 files changed, 91 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java ---------------------------------------------------------------------- diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index d223cdf..cf36f61 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -67,6 +67,7 @@ import org.apache.calcite.rel.stream.Delta; import org.apache.calcite.rel.stream.LogicalDelta; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -194,6 +195,7 @@ import static org.apache.calcite.util.Static.RESOURCE; * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE` * - isTrimUnusedFields(), override to false * - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104 + * - convertQuery(), call hackSelectStar() at the end */ /** @@ -529,6 +531,8 @@ public class SqlToRelConverter { * the query will be part of a view. */ public RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean top) { + SqlNode origQuery = query; /* OVERRIDE POINT */ + if (needsValidation) { query = validator.validate(query); } @@ -553,7 +557,63 @@ public class SqlToRelConverter { } final RelDataType validatedRowType = validator.getValidatedNodeType(query); - return RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation); + return hackSelectStar(origQuery, RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation)); + } + + /* OVERRIDE POINT */ + private RelRoot hackSelectStar(SqlNode query, RelRoot root) { + /* + * Rel tree is like: + * + * LogicalSort (optional) + * |- LogicalProject + * |- OLAPTableScan + */ + LogicalProject rootPrj = null; + LogicalSort rootSort = null; + if (root.rel instanceof LogicalProject) { + rootPrj = (LogicalProject) root.rel; + } else if (root.rel instanceof LogicalSort && root.rel.getInput(0) instanceof LogicalProject) { + rootPrj = (LogicalProject) root.rel.getInput(0); + rootSort = (LogicalSort) root.rel; + } else { + return root; + } + + if (!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan")) + return root; + + RelNode scan = rootPrj.getInput(); + if (rootPrj.getRowType().getFieldCount() < scan.getRowType().getFieldCount()) + return root; + + RelDataType inType = rootPrj.getRowType(); + List<String> inFields = inType.getFieldNames(); + List<RexNode> projExp = new ArrayList<>(); + List<Pair<Integer, String>> projFields = new ArrayList<>(); + FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder(); + FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder(); + for (int i = 0; i < inFields.size(); i++) { + if (!inFields.get(i).startsWith("_KY_")) { + projExp.add(rootPrj.getProjects().get(i)); + projFields.add(Pair.of(projFields.size(), inFields.get(i))); + projTypeBuilder.add(inType.getFieldList().get(i)); + validTypeBuilder.add(root.validatedRowType.getFieldList().get(i)); + } + } + + RelDataType projRowType = getCluster().getTypeFactory().createStructType(projTypeBuilder); + rootPrj = LogicalProject.create(scan, projExp, projRowType); + if (rootSort != null) { + rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch); + } + + RelDataType validRowType = getCluster().getTypeFactory().createStructType(validTypeBuilder); + root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind, projFields, root.collation); + + validator.setValidatedNodeType(query, validRowType); + + return root; } private static boolean isStream(SqlNode query) { http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index ac13f40..4d89e1a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -138,9 +138,9 @@ public class FunctionDesc { if (isSum()) { return getParameter().getValue(); } else if (isCount()) { - return "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same + return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same } else { - return getFullExpression().replaceAll("[(),. ]", "_"); + return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index d485955..7f04bbb 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -379,4 +379,10 @@ public class ITKylinQueryTest extends KylinTestBase { // compare the result Assert.assertEquals(expectVersion, queriedVersion); } + + @Test + public void testSelectStarColumnCount() throws Exception { + execAndCompColumnCount("select * from test_kylin_fact limit 10", 9); + execAndCompColumnCount("select * from test_kylin_fact", 9); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index bcf55e5..ddb996c 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -67,6 +67,7 @@ import org.dbunit.ext.h2.H2Connection; import org.dbunit.ext.h2.H2DataTypeFactory; import org.junit.Assert; +import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; /** @@ -422,6 +423,26 @@ public class KylinTestBase { } } + protected void execAndCompColumnCount(String input, int expectedColumnCount) throws Exception { + printInfo("---------- test column count: " + input); + Set<String> sqlSet = ImmutableSet.of(input); + + for (String sql : sqlSet) { + // execute Kylin + printInfo("Query Result from Kylin - " + sql); + IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection); + ITable kylinTable = executeQuery(kylinConn, sql, sql, false); + + try { + // compare the result + Assert.assertEquals(expectedColumnCount, kylinTable.getTableMetaData().getColumns().length); + } catch (Throwable t) { + printInfo("execAndCompColumnCount failed on: " + sql); + throw t; + } + } + } + protected void execLimitAndValidate(String queryFolder) throws Exception { printInfo("---------- test folder: " + new File(queryFolder).getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/kylin-it/src/test/resources/query/sql_timeout/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_timeout/query01.sql b/kylin-it/src/test/resources/query/sql_timeout/query01.sql index 3b9a837..eaff396 100644 --- a/kylin-it/src/test/resources/query/sql_timeout/query01.sql +++ b/kylin-it/src/test/resources/query/sql_timeout/query01.sql @@ -16,4 +16,4 @@ -- limitations under the License. -- -select * from test_kylin_fact limit 1200 +select seller_id,lstg_format_name from test_kylin_fact limit 1200
