Revert "KYLIN-2816 wrap columns with backtick" This reverts commit 53b25c6137731c1b0d26925be7dcaa0252d75b64.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81e9f573 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81e9f573 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81e9f573 Branch: refs/heads/2622-2764 Commit: 81e9f573fae2af9511ac6004f69101598fc7a71a Parents: dbecb9b Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Aug 28 19:19:10 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Tue Aug 29 10:36:10 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/job/JoinedFlatTable.java | 77 ++------------------ .../kylin/job/TestWrapColumnWithBacktick.java | 57 --------------- .../metadata/model/tool/CalciteParser.java | 8 +- 3 files changed, 7 insertions(+), 135 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/81e9f573/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 4ac5a3a..824c693 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -21,17 +21,11 @@ package org.apache.kylin.job; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.metadata.model.DataModelDesc; @@ -41,9 +35,6 @@ import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.PartitionDesc; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.model.tool.CalciteParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -52,7 +43,6 @@ import org.slf4j.LoggerFactory; public class JoinedFlatTable { public static final String BACKTICK = "`"; - private static final Logger logger = LoggerFactory.getLogger(JoinedFlatTable.class); public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); @@ -140,7 +130,11 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); - String expressionInSourceDB = quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB()); + String expressionInSourceDB = col.getExpressionInSourceDB(); + if (expressionInSourceDB.contains(".")) { + // surround column name with back-tick, to support unicode column name + expressionInSourceDB = expressionInSourceDB.replace(".", "." + BACKTICK) + BACKTICK; + } if (skipAsList.contains(colTotalName)) { sql.append(expressionInSourceDB + sep); } else { @@ -283,65 +277,4 @@ public class JoinedFlatTable { return sql.toString(); } - /** - * quote column name with back-tick, to support unicode column name - * - * @param sourceSQL - * @return - */ - static String quoteOriginalColumnWithBacktick(String sourceSQL) { - StringBuilder result = new StringBuilder(sourceSQL); - try { - List<Pair<Integer, Integer>> replacePoses = new ArrayList<>(); - for (SqlIdentifier id : SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) { - replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1), "select " + sourceSQL + " from t")); - } - - // latter replace position in the front of the list. - Collections.sort(replacePoses, new Comparator<Pair<Integer, Integer>>() { - @Override - public int compare(Pair<Integer, Integer> o1, Pair<Integer, Integer> o2) { - return -(o1.getSecond() - o2.getSecond()); - } - }); - - // cuz the pos is add "select "'s size, so minus 7 offset. - final int OFFSET = 7; - for (Pair<Integer, Integer> replacePos : replacePoses) { - result.insert(replacePos.getSecond() - OFFSET, "`"); - result.insert(replacePos.getFirst() - OFFSET, "`"); - } - } catch (Exception e) { - logger.error("quote original column with backtick fail.Return origin SQL." + e.getMessage()); - } - - return result.toString(); - } - - - static class SqlIdentifierVisitor extends SqlBasicVisitor<SqlNode> { - private List<SqlIdentifier> sqlIdentifier; - - SqlIdentifierVisitor() { - this.sqlIdentifier = new ArrayList<>(); - } - - List<SqlIdentifier> getSqlIdentifier() { - return sqlIdentifier; - } - - static List<SqlIdentifier> getAllSqlIdentifier(SqlNode node) { - SqlIdentifierVisitor siv = new SqlIdentifierVisitor(); - node.accept(siv); - return siv.getSqlIdentifier(); - } - - @Override - public SqlNode visit(SqlIdentifier id) { - if (id.names.size() == 2) { - sqlIdentifier.add(id); - } - return null; - } - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/81e9f573/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java ---------------------------------------------------------------------- diff --git a/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java b/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java deleted file mode 100644 index 0c55ba7..0000000 --- a/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.job; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class TestWrapColumnWithBacktick { - @Test - public void testWrapColumnNameInFlatTable() { - String input1 = "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"; - String expected1 = "SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`"; - String input2 = "year(TEST_KYLIN_FACT.CAL_DT)"; - String expected2 = "year(TEST_KYLIN_FACT.`CAL_DT`)"; - String input3 = "SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1)"; - String expected3 = "SUBSTR(SELLER_ACCOUNT.`ACCOUNT_COUNTRY`,0,1)"; - String input4 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)"; - String expected4 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, SELLER_COUNTRY.`NAME`)"; - String input5 = "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL*1.1"; - String expected5 = "SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`*1.1"; - String input6 = "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL*SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"; - String expected6 = "SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`*SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`"; - String input7 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, 'HELLO.WORLD')"; - String expected7 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, 'HELLO.WORLD')"; - String input8 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_ACCOUNT.ACCOUNT_ID2)"; - String expected8 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, SELLER_ACCOUNT.`ACCOUNT_ID2`)"; - String input9 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_ACCOUNT.ACCOUNT_ID2, SELLER_ACCOUNT.ACCOUNT_ID3)"; - String expected9 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, SELLER_ACCOUNT.`ACCOUNT_ID2`, SELLER_ACCOUNT.`ACCOUNT_ID3`)"; - - assertEquals(expected1, JoinedFlatTable.quoteOriginalColumnWithBacktick(input1)); - assertEquals(expected2, JoinedFlatTable.quoteOriginalColumnWithBacktick(input2)); - assertEquals(expected3, JoinedFlatTable.quoteOriginalColumnWithBacktick(input3)); - assertEquals(expected4, JoinedFlatTable.quoteOriginalColumnWithBacktick(input4)); - assertEquals(expected5, JoinedFlatTable.quoteOriginalColumnWithBacktick(input5)); - assertEquals(expected6, JoinedFlatTable.quoteOriginalColumnWithBacktick(input6)); - assertEquals(expected7, JoinedFlatTable.quoteOriginalColumnWithBacktick(input7)); - assertEquals(expected8, JoinedFlatTable.quoteOriginalColumnWithBacktick(input8)); - assertEquals(expected9, JoinedFlatTable.quoteOriginalColumnWithBacktick(input9)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/81e9f573/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java index 09f2f28..fdad33a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java @@ -32,11 +32,11 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlVisitor; -import org.apache.kylin.common.util.Pair; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.kylin.common.util.Pair; public class CalciteParser { public static SqlNode parse(String sql) throws SqlParseException { @@ -127,15 +127,11 @@ public class CalciteParser { } public static Pair<Integer, Integer> getReplacePos(SqlNode node, String inputSql) { - SqlParserPos pos = node.getParserPosition(); - return getReplacePos(pos, inputSql); - } - - public static Pair<Integer, Integer> getReplacePos(SqlParserPos pos, String inputSql) { if (inputSql == null) { return Pair.newPair(0, 0); } String[] lines = inputSql.split("\n"); + SqlParserPos pos = node.getParserPosition(); int lineStart = pos.getLineNum(); int lineEnd = pos.getEndLineNum(); int columnStart = pos.getColumnNum() - 1;
