KYLIN-1954 refind and fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdad2d8d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdad2d8d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdad2d8d Branch: refs/heads/1.5.x-CDH5.7 Commit: fdad2d8d3ff34b069f646794226bcb73543a9474 Parents: cc9acbc Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Aug 30 09:55:57 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Aug 30 11:05:26 2016 +0800 ---------------------------------------------------------------------- .../kylin/job/dataGen/FactTableGenerator.java | 10 +++-- .../kylin/dict/BuildInFunctionTransformer.java | 5 ++- .../localmeta/data/data_gen_config.json | 2 +- .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 4 +- .../apache/kylin/query/ITCombinationTest.java | 14 ++++-- .../apache/kylin/query/ITKylinQueryTest.java | 13 +++++- .../test/resources/query/sql_like/query01.sql | 31 +++++++++++++ .../test/resources/query/sql_like/query02.sql | 22 +++++++++ .../apache/kylin/query/routing/RoutingRule.java | 2 + .../rules/RemoveBlackoutRealizationsRule.java | 47 ++++++++++++++++++++ 10 files changed, 136 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java index 9373a02..5aac4de 100644 --- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java +++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java @@ -586,10 +586,12 @@ public class FactTableGenerator { } for (Integer index : differentiateColumns) { - if (currentRowTime >= differentiateBoundary) { - columnValues.set(index, columnValues.get(index) + "_B"); - } else { - columnValues.set(index, columnValues.get(index) + "_A"); + if (r.nextBoolean()) {//only change half of data + if (currentRowTime >= differentiateBoundary) { + columnValues.set(index, columnValues.get(index) + "_B"); + } else { + columnValues.set(index, columnValues.get(index) + "_A"); + } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java index 9156b67..665a7e6 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java @@ -70,8 +70,11 @@ public class BuildInFunctionTransformer implements ITupleFilterTransformer { ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); while (childIterator.hasNext()) { TupleFilter transformed = transform(childIterator.next()); - if (transformed != null) + if (transformed != null) { childIterator.set(transformed); + } else { + throw new IllegalStateException("Should not be null"); + } } } return translated == null ? tupleFilter : translated; http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/examples/test_case_data/localmeta/data/data_gen_config.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json index f730058..16ba3d5 100644 --- a/examples/test_case_data/localmeta/data/data_gen_config.json +++ b/examples/test_case_data/localmeta/data/data_gen_config.json @@ -1,5 +1,5 @@ { - "differentiateBoundary": "2013-01-01", + "differentiateBoundary": "2013-07-01", "columnConfigs": [ { "columnName": "lstg_format_name", http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java index ca3a9bc..4c847bf 100644 --- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java @@ -217,7 +217,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase { PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME"); - statement.setString(1, "FP-GTC"); + statement.setString(1, "FP-GTC_A"); ResultSet rs = statement.executeQuery(); @@ -225,7 +225,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase { String format_name = rs.getString(1); - Assert.assertTrue("FP-GTC".equals(format_name)); + Assert.assertTrue("FP-GTC_A".equals(format_name)); rs.close(); statement.close(); http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index cf18b20..cbd4e44 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.routing.Candidate; +import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; import org.apache.kylin.storage.hbase.HBaseStorage; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; import org.junit.AfterClass; @@ -64,13 +65,12 @@ public class ITCombinationTest extends ITKylinQueryTest { */ @Parameterized.Parameters public static Collection<Object[]> configs() { - // return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, }); - return Arrays.asList(new Object[][] { { "inner", "on", "v2" }, { "left", "on", "v1" }, { "left", "on", "v2" } }); + return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on", "v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on", "v2", true } }); } - public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine) throws Exception { + public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception { - printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine); + printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine + ", excludeViewCubes: " + excludeViewCubes); ITKylinQueryTest.clean(); @@ -85,6 +85,12 @@ public class ITCombinationTest extends ITKylinQueryTest { // unset } + RemoveBlackoutRealizationsRule.blackouts.clear(); + if (excludeViewCubes) { + RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]"); + RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]"); + } + if ("v1".equalsIgnoreCase(queryEngine)) HBaseStorage.overwriteStorageQuery = HBaseStorage.v1CubeStorageQuery; else http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index dcacb06..4657a5c 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -34,6 +34,7 @@ import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.routing.Candidate; +import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; import org.apache.kylin.query.schema.OLAPSchemaFactory; import org.apache.kylin.storage.hbase.HBaseStorage; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; @@ -46,7 +47,6 @@ import org.junit.Test; import com.google.common.collect.Maps; -@Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { @BeforeClass @@ -57,9 +57,12 @@ public class ITKylinQueryTest extends KylinTestBase { priorities.put(RealizationType.CUBE, 0); Candidate.setPriorities(priorities); - joinType = "left"; + joinType = "inner"; setupAll(); + + RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]"); + RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]"); } @AfterClass @@ -96,6 +99,7 @@ public class ITKylinQueryTest extends KylinTestBase { ObserverEnabler.forceCoprocessorUnset(); HBaseMetadataTestCase.staticCleanupTestMetadata(); + RemoveBlackoutRealizationsRule.blackouts.clear(); } @@ -147,6 +151,11 @@ public class ITKylinQueryTest extends KylinTestBase { } @Test + public void testLikeQuery() throws Exception { + execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_like", null, true); + } + + @Test public void testVerifyQuery() throws Exception { verifyResultRowCount(getQueryFolderPrefix() + "src/test/resources/query/sql_verifyCount"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql new file mode 100644 index 0000000..e1f131d --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query01.sql @@ -0,0 +1,31 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select lstg_format_name as lstg_format_name, count(*) as cnt + + from test_kylin_fact +inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + inner JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + +where lstg_format_name like '%BIN%' +group by lstg_format_name \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query02.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_like/query02.sql b/kylin-it/src/test/resources/query/sql_like/query02.sql new file mode 100644 index 0000000..ef0b4a0 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_like/query02.sql @@ -0,0 +1,22 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact +where lower(lstg_format_name)='abin_a' and substring(lstg_format_name,1,3) in ('ABI') and upper(lstg_format_name) > 'AAAA' and +lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='ABIN_Aa' +group by lstg_format_name \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java index 11498d6..53f871d 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java +++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.query.routing.rules.RealizationSortRule; +import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; import org.apache.kylin.query.routing.rules.RemoveUncapableRealizationsRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +38,7 @@ public abstract class RoutingRule { private static List<RoutingRule> rules = Lists.newLinkedList(); static { + rules.add(new RemoveBlackoutRealizationsRule()); rules.add(new RemoveUncapableRealizationsRule()); rules.add(new RealizationSortRule()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java new file mode 100644 index 0000000..9c3d7c9 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.query.routing.rules; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.kylin.query.routing.Candidate; +import org.apache.kylin.query.routing.RoutingRule; + +import com.google.common.collect.Sets; + +/** + * for IT use, exclude some cubes + */ +public class RemoveBlackoutRealizationsRule extends RoutingRule { + public static Set<String> blackouts = Sets.newHashSet(); + + @Override + public void apply(List<Candidate> candidates) { + for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) { + Candidate candidate = iterator.next(); + + if (blackouts.contains(candidate.getRealization().getCanonicalName())) { + iterator.remove(); + } + } + } + +}
