KYLIN-1475 Inject ehcache manager for any test case that will touch ehcache manager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f49c34 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f49c34 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f49c34 Branch: refs/heads/1.4-rc Commit: 83f49c34f0302bfcc323a760bf6097051006546f Parents: e177ef5 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Mar 8 14:30:56 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Mar 8 14:30:56 2016 +0800 ---------------------------------------------------------------------- .../dict/TupleFilterDictionaryTranslater.java | 166 ------------------- .../dict/TupleFilterFunctionTransformer.java | 161 ++++++++++++++++++ .../filter/ITupleFilterTransformer.java | 23 +++ .../metadata/filter/ITupleFilterTranslator.java | 26 --- .../metadata/filter/function/BuiltInMethod.java | 24 ++- .../cache/AbstractCacheFledgedQuery.java | 29 +--- .../kylin/storage/cache/DynamicCacheTest.java | 31 +++- .../kylin/storage/cache/StaticCacheTest.java | 32 +++- .../apache/kylin/query/ITKylinQueryTest.java | 15 ++ .../kylin/storage/hbase/ITStorageTest.java | 22 ++- .../common/coprocessor/FilterDecorator.java | 6 +- .../hbase/cube/v2/CubeSegmentScanner.java | 6 +- 12 files changed, 285 insertions(+), 256 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java deleted file mode 100644 index fec6a85..0000000 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import com.google.common.primitives.Primitives; -import org.apache.kylin.dict.Dictionary; -import org.apache.kylin.dict.IDictionaryAware; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.FunctionTupleFilter; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ListIterator; - -/** - * Created by dongli on 1/7/16. - */ -public class TupleFilterDictionaryTranslater implements ITupleFilterTranslator { - public static final Logger logger = LoggerFactory.getLogger(TupleFilterDictionaryTranslater.class); - - private IDictionaryAware dictionaryAware; - - public TupleFilterDictionaryTranslater(IDictionaryAware dictionaryAware) { - this.dictionaryAware = dictionaryAware; - } - - @Override - public TupleFilter translate(TupleFilter tupleFilter) { - TupleFilter translated = null; - if (tupleFilter instanceof CompareTupleFilter) { - translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof FunctionTupleFilter) { - translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof LogicalTupleFilter) { - ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); - while (childIterator.hasNext()) { - TupleFilter tempTranslated = translate(childIterator.next()); - if (tempTranslated != null) - childIterator.set(tempTranslated); - } - } - return translated == null ? tupleFilter : translated; - } - - private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { - if (!functionTupleFilter.isValid()) - return null; - - TblColRef columnRef = functionTupleFilter.getColumn(); - Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } - - @SuppressWarnings("unchecked") - private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { - if (compTupleFilter.getFunction() == null) - return null; - - FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction(); - if (!functionTupleFilter.isValid()) - return null; - - TblColRef columnRef = functionTupleFilter.getColumn(); - Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - Object computedVal = functionTupleFilter.invokeFunction(dictVal); - Class clazz = Primitives.wrap(computedVal.getClass()); - Object targetVal = compTupleFilter.getFirstValue(); - if (Primitives.isWrapperType(clazz)) - targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); - - int comp = ((Comparable) computedVal).compareTo(targetVal); - boolean compResult = false; - switch (compTupleFilter.getOperator()) { - case EQ: - compResult = comp == 0; - break; - case NEQ: - compResult = comp != 0; - break; - case LT: - compResult = comp < 0; - break; - case LTE: - compResult = comp <= 0; - break; - case GT: - compResult = comp > 0; - break; - case GTE: - compResult = comp >= 0; - break; - case IN: - compResult = compTupleFilter.getValues().contains(computedVal.toString()); - break; - case NOTIN: - compResult = !compTupleFilter.getValues().contains(computedVal.toString()); - break; - default: - break; - } - if (compResult) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java new file mode 100644 index 0000000..eadc291 --- /dev/null +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dict; + +import com.google.common.primitives.Primitives; +import org.apache.kylin.metadata.filter.ColumnTupleFilter; +import org.apache.kylin.metadata.filter.CompareTupleFilter; +import org.apache.kylin.metadata.filter.ConstantTupleFilter; +import org.apache.kylin.metadata.filter.FunctionTupleFilter; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; +import org.apache.kylin.metadata.filter.LogicalTupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ListIterator; + +public class TupleFilterFunctionTransformer implements ITupleFilterTransformer { + public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTransformer.class); + + private IDictionaryAware dictionaryAware; + + public TupleFilterFunctionTransformer(IDictionaryAware dictionaryAware) { + this.dictionaryAware = dictionaryAware; + } + + @Override + public TupleFilter translate(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof FunctionTupleFilter) { + translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof LogicalTupleFilter) { + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter tempTranslated = translate(childIterator.next()); + if (tempTranslated != null) + childIterator.set(tempTranslated); + } + } + return translated == null ? tupleFilter : translated; + } + + private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } + + @SuppressWarnings("unchecked") + private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { + if (compTupleFilter.getFunction() == null) + return null; + + FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction(); + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + Object computedVal = functionTupleFilter.invokeFunction(dictVal); + Class clazz = Primitives.wrap(computedVal.getClass()); + Object targetVal = compTupleFilter.getFirstValue(); + if (Primitives.isWrapperType(clazz)) + targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); + + int comp = ((Comparable) computedVal).compareTo(targetVal); + boolean compResult = false; + switch (compTupleFilter.getOperator()) { + case EQ: + compResult = comp == 0; + break; + case NEQ: + compResult = comp != 0; + break; + case LT: + compResult = comp < 0; + break; + case LTE: + compResult = comp <= 0; + break; + case GT: + compResult = comp > 0; + break; + case GTE: + compResult = comp >= 0; + break; + case IN: + compResult = compTupleFilter.getValues().contains(computedVal.toString()); + break; + case NOTIN: + compResult = !compTupleFilter.getValues().contains(computedVal.toString()); + break; + default: + break; + } + if (compResult) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java new file mode 100644 index 0000000..05fa7b0 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +public interface ITupleFilterTransformer { + TupleFilter translate(TupleFilter tupleFilter); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java deleted file mode 100644 index aed284c..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.metadata.filter; - -/** - * Created by dongli on 1/7/16. - */ -public interface ITupleFilterTranslator { - TupleFilter translate(TupleFilter tupleFilter); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java index b927d8d..97bca3b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java @@ -53,13 +53,17 @@ public enum BuiltInMethod { this.method = MethodUtils.getMatchingAccessibleMethod(clazz, methodName, argumentTypes); } - /** SQL {@code LIKE} function. */ + /** + * SQL {@code LIKE} function. + */ public static boolean like(String s, String pattern) { final String regex = Like.sqlToRegexLike(pattern, null); return Pattern.matches(regex, s); } - /** SQL INITCAP(string) function. */ + /** + * SQL INITCAP(string) function. + */ public static String initcap(String s) { // Assumes Alpha as [A-Za-z0-9] // white space is treated as everything else. @@ -96,22 +100,30 @@ public enum BuiltInMethod { return newS.toString(); } - /** SQL CHARACTER_LENGTH(string) function. */ + /** + * SQL CHARACTER_LENGTH(string) function. + */ public static int charLength(String s) { return s.length(); } - /** SQL SUBSTRING(string FROM ... FOR ...) function. */ + /** + * SQL SUBSTRING(string FROM ... FOR ...) function. + */ public static String substring(String s, int from, int for_) { return s.substring(from - 1, Math.min(from - 1 + for_, s.length())); } - /** SQL UPPER(string) function. */ + /** + * SQL UPPER(string) function. + */ public static String upper(String s) { return s.toUpperCase(); } - /** SQL LOWER(string) function. */ + /** + * SQL LOWER(string) function. + */ public static String lower(String s) { return s.toLowerCase(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java index 083021e..7826c0c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java @@ -55,31 +55,6 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup CACHE_MANAGER = cacheManager; } - /** - * This method is only useful non-spring injected test cases. - * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected. - * and the configuration for cache lies in server/src/main/resources/ehcache.xml - * - * the cache named "StorageCache" acts like a template for each realization to - * create its own cache. - */ - private static void initCacheManger() { - Configuration conf = new Configuration(); - conf.setMaxBytesLocalHeap("128M"); - CACHE_MANAGER = CacheManager.create(conf); - - //a fake template for test cases - Cache storageCache = new Cache(new CacheConfiguration(storageCacheTemplate, 0).// - memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).// - eternal(false).// - timeToIdleSeconds(86400).// - diskExpiryThreadIntervalSeconds(0).// - //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).// - persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE))); - - CACHE_MANAGER.addCacheIfAbsent(storageCache); - } - protected StreamSQLResult getStreamSQLResult(StreamSQLDigest streamSQLDigest) { Cache cache = CACHE_MANAGER.getCache(this.underlyingStorage.getStorageUUID()); @@ -105,10 +80,10 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup private void makeCacheIfNecessary(String storageUUID) { if (CACHE_MANAGER == null || (!(CACHE_MANAGER.getStatus().equals(Status.STATUS_ALIVE)))) { - logger.warn("CACHE_MANAGER is not provided or not alive"); - initCacheManger(); + throw new RuntimeException("CACHE_MANAGER is not provided or not alive"); } + if (CACHE_MANAGER.getCache(storageUUID) == null) { logger.info("Cache for {} initializing...", storageUUID); http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java index c9ea479..732f5f6 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java @@ -18,11 +18,10 @@ package org.apache.kylin.storage.cache; -import java.util.ArrayList; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - +import com.google.common.collect.Lists; +import com.google.common.collect.Range; +import com.google.common.collect.Ranges; +import net.sf.ehcache.CacheManager; import org.apache.commons.lang.NotImplementedException; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.DateFormat; @@ -38,22 +37,36 @@ import org.apache.kylin.storage.ICachableStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.tuple.Tuple; import org.apache.kylin.storage.tuple.TupleInfo; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import com.google.common.collect.Lists; -import com.google.common.collect.Range; -import com.google.common.collect.Ranges; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; /** */ public class DynamicCacheTest { + private static CacheManager cacheManager; + + @BeforeClass public static void setup() { System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cache.threshold.duration", "0"); + + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); + } + + @AfterClass + public static void tearDownResource() { + cacheManager.shutdown(); + AbstractCacheFledgedQuery.setCacheManager(null); } @@ -105,7 +118,7 @@ public class DynamicCacheTest { final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations(); final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations); - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef> newArrayList(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); + SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef>newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef>newArrayList(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01"); ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01"); http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java index 3b50736..bdbb378 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java @@ -18,12 +18,9 @@ package org.apache.kylin.storage.cache; -import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - +import com.google.common.collect.Lists; +import com.google.common.collect.Range; +import net.sf.ehcache.CacheManager; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.IdentityUtils; import org.apache.kylin.metadata.filter.TupleFilter; @@ -38,22 +35,39 @@ import org.apache.kylin.storage.ICachableStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.tuple.Tuple; import org.apache.kylin.storage.tuple.TupleInfo; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import com.google.common.collect.Lists; -import com.google.common.collect.Range; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; /** */ public class StaticCacheTest { + private static CacheManager cacheManager; + + @BeforeClass public static void setup() { System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cache.threshold.duration", "0"); + + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); } + @AfterClass + public static void tearDownResource() { + cacheManager.shutdown(); + AbstractCacheFledgedQuery.setCacheManager(null); + } + + @Test public void basicTest() { @@ -61,7 +75,7 @@ public class StaticCacheTest { final List<TblColRef> groups = StorageMockUtils.buildGroups(); final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations(); final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0)); - final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); + final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef>emptySet(), groups, Collections.<TblColRef>emptySet(), Collections.<TblColRef>emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations); final List<ITuple> ret = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index d7dd260..c400929 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -25,12 +25,14 @@ import java.sql.DriverManager; import java.util.List; import java.util.Properties; +import net.sf.ehcache.CacheManager; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.schema.OLAPSchemaFactory; +import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.dbunit.database.DatabaseConnection; @@ -42,6 +44,8 @@ import org.junit.Test; @Ignore("KylinQueryTest is contained by ITCombinationTest") public class ITKylinQueryTest extends KylinTestBase { + private static CacheManager cacheManager; + @BeforeClass public static void setUp() throws Exception { @@ -74,6 +78,11 @@ public class ITKylinQueryTest extends KylinTestBase { // Load H2 Tables (inner join) H2Database h2DB = new H2Database(h2Connection, config); h2DB.loadAllTables(); + + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); + + } protected static void clean() { @@ -84,6 +93,12 @@ public class ITKylinQueryTest extends KylinTestBase { ObserverEnabler.forceCoprocessorUnset(); HBaseMetadataTestCase.staticCleanupTestMetadata(); + + if (cacheManager != null) { + cacheManager.shutdown(); + } + AbstractCacheFledgedQuery.setCacheManager(null); + } @Ignore("this is only for debug") http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index 5ce26c7..5a24d28 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -18,12 +18,7 @@ package org.apache.kylin.storage.hbase; -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - +import net.sf.ehcache.CacheManager; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.cube.CubeInstance; @@ -38,6 +33,7 @@ import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.StorageFactory; +import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; import org.apache.kylin.storage.cache.StorageMockUtils; import org.apache.kylin.storage.exception.ScanOutOfLimitException; import org.junit.After; @@ -48,7 +44,15 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertTrue; + public class ITStorageTest extends HBaseMetadataTestCase { + private static CacheManager cacheManager; + private IStorageQuery storageEngine; private CubeInstance cube; @@ -56,10 +60,14 @@ public class ITStorageTest extends HBaseMetadataTestCase { @BeforeClass public static void setupResource() throws Exception { + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); } @AfterClass public static void tearDownResource() { + cacheManager.shutdown(); + AbstractCacheFledgedQuery.setCacheManager(null); } @Before @@ -144,7 +152,7 @@ public class ITStorageTest extends HBaseMetadataTestCase { int count = 0; ITupleIterator iterator = null; try { - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); + SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef>emptySet(), groups, Collections.<TblColRef>emptySet(), Collections.<TblColRef>emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations)); while (iterator.hasNext()) { ITuple tuple = iterator.next(); http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java index cb73b22..ab6c8b8 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -26,11 +26,11 @@ import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.dict.DictCodeSystem; import org.apache.kylin.dict.Dictionary; import org.apache.kylin.dict.IDictionaryAware; -import org.apache.kylin.dict.TupleFilterDictionaryTranslater; +import org.apache.kylin.dict.TupleFilterFunctionTransformer; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.filter.TupleFilterSerializer; import org.apache.kylin.metadata.model.TblColRef; @@ -149,7 +149,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { if (filter == null) return null; - ITupleFilterTranslator translator = new TupleFilterDictionaryTranslater(columnIO.getIDictionaryAware()); + ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(columnIO.getIDictionaryAware()); filter = translator.translate(filter); // un-evaluatable filter is replaced with TRUE http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index 6f3d012..55b9b15 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -41,7 +41,7 @@ import org.apache.kylin.cube.gridtable.CubeGridTable; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.dict.TupleFilterDictionaryTranslater; +import org.apache.kylin.dict.TupleFilterFunctionTransformer; import org.apache.kylin.gridtable.EmptyGTScanner; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; @@ -50,7 +50,7 @@ import org.apache.kylin.gridtable.GTScanRangePlanner; import org.apache.kylin.gridtable.GTScanRequest; import org.apache.kylin.gridtable.GTUtil; import org.apache.kylin.gridtable.IGTScanner; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; +import org.apache.kylin.metadata.filter.ITupleFilterTransformer; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.DataType; import org.apache.kylin.metadata.model.FunctionDesc; @@ -82,7 +82,7 @@ public class CubeSegmentScanner implements IGTScanner { CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping(); // translate FunctionTupleFilter to IN clause - ITupleFilterTranslator translator = new TupleFilterDictionaryTranslater(this.cubeSeg); + ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(this.cubeSeg); filter = translator.translate(filter); //replace the constant values in filter to dictionary codes
