KYLIN-2255 cleanup in StorageEngine, pass CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/940d84f2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/940d84f2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/940d84f2 Branch: refs/heads/master-cdh5.7 Commit: 940d84f237652a35f5d7debb05115ee13a938aa5 Parents: c31c849 Author: Yang Li <liy...@apache.org> Authored: Thu Dec 8 21:40:54 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Fri Dec 9 06:51:48 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/query/ITCombinationTest.java | 2 -- .../apache/kylin/query/ITKylinQueryTest.java | 37 +++++++------------- .../resources/query/sql_subquery/query02.sql | 36 +++++++++++++++++++ .../query/sql_subquery/query02.sql.disabled | 31 ---------------- .../kylin/storage/hbase/HBaseStorage.java | 8 +---- 5 files changed, 50 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index 84573b5..5f1e3a9 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.routing.Candidate; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; -import org.apache.kylin.storage.hbase.HBaseStorage; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.runner.RunWith; @@ -53,7 +52,6 @@ public class ITCombinationTest extends ITKylinQueryTest { public static void tearDown() { printInfo("tearDown in ITCombinationTest"); clean(); - HBaseStorage.overwriteStorageQuery = null; Candidate.restorePriorities(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 90324b5..d485955 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -34,7 +34,6 @@ import org.apache.kylin.gridtable.StorageSideBehavior; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.routing.Candidate; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; -import org.apache.kylin.storage.hbase.HBaseStorage; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; import org.dbunit.dataset.ITable; @@ -95,10 +94,6 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testTimeoutQuery() throws Exception { - if (HBaseStorage.overwriteStorageQuery != null) { - //v1 engine does not suit - return; - } try { Map<String, String> toggles = Maps.newHashMap(); @@ -157,7 +152,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testSingleRunQuery() throws Exception { - String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql"; + String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query02.sql"; File sqlFile = new File(queryFileName); if (sqlFile.exists()) { @@ -329,30 +324,25 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testLimitEnabled() throws Exception { - if (HBaseStorage.overwriteStorageQuery == null) {//v1 query engine will not work - - try { - //other cubes have strange aggregation groups - RemoveBlackoutRealizationsRule.whiteList.add("CUBE[name=test_kylin_cube_with_slr_empty]"); - - List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql"); - for (File sqlFile : sqlFiles) { - runSQL(sqlFile, false, false); - assertTrue(checkLimitEnabled()); - assertTrue(checkFinalPushDownLimit()); - } + try { + //other cubes have strange aggregation groups + RemoveBlackoutRealizationsRule.whiteList.add("CUBE[name=test_kylin_cube_with_slr_empty]"); - } finally { - RemoveBlackoutRealizationsRule.whiteList.remove("CUBE[name=test_kylin_cube_with_slr_empty]"); + List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql"); + for (File sqlFile : sqlFiles) { + runSQL(sqlFile, false, false); + assertTrue(checkLimitEnabled()); + assertTrue(checkFinalPushDownLimit()); } + + } finally { + RemoveBlackoutRealizationsRule.whiteList.remove("CUBE[name=test_kylin_cube_with_slr_empty]"); } } @Test public void testLimitCorrectness() throws Exception { - if (HBaseStorage.overwriteStorageQuery == null) {//v1 query engine will not work - execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql"); - } + this.execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql"); } @Test @@ -385,7 +375,6 @@ public class ITKylinQueryTest extends KylinTestBase { IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection); ITable kylinTable = executeQuery(kylinConn, queryName, sql, false); String queriedVersion = String.valueOf(kylinTable.getValue(0, "version")); - // compare the result Assert.assertEquals(expectVersion, queriedVersion); http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/resources/query/sql_subquery/query02.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql new file mode 100644 index 0000000..3ecfb1b --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql @@ -0,0 +1,36 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +SELECT + week_beg_dt + ,sum(price) as sum_price +FROM +( + select + test_cal_dt.week_beg_dt + ,test_kylin_fact.price + from test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id +) t +group by week_beg_dt +order by week_beg_dt \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled b/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled deleted file mode 100644 index e6751b7..0000000 --- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled +++ /dev/null @@ -1,31 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -SELECT - week_beg_dt - ,sum(price) as sum_price -FROM -( - select - test_cal_dt.week_beg_dt - ,test_kylin_fact.price - from test_kylin_fact - inner join edw.test_cal_dt as test_cal_dt - on test_kylin_fact.cal_dt = test_cal_dt.cal_dt -) t -group by week_beg_dt http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java index 43b65cb..5e8c620 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java @@ -41,8 +41,6 @@ import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition; public class HBaseStorage implements IStorage { public final static String v2CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery"; - public final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery"; - public static String overwriteStorageQuery = null;//for test case @Override public IStorageQuery createQuery(IRealization realization) { @@ -52,11 +50,7 @@ public class HBaseStorage implements IStorage { CubeInstance cubeInstance = (CubeInstance) realization; String cubeStorageQuery; if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now - cubeStorageQuery = v1CubeStorageQuery; - } else if (overwriteStorageQuery != null) { - cubeStorageQuery = overwriteStorageQuery; - } else if ("v1".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryVersion())) { - cubeStorageQuery = v1CubeStorageQuery; + throw new IllegalStateException("Storage Engine (id=" + IStorageAware.ID_HBASE + ") is not supported any more"); } else { cubeStorageQuery = v2CubeStorageQuery;//by default use v2 }
