minor, add LIKE FilterOperatorEnum in case it will be pushed down(it won't for now)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e0b8d80 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e0b8d80 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e0b8d80 Branch: refs/heads/1.5.x-CDH5.7 Commit: 9e0b8d8033dbe4e86d299adbd102cf52792cd5fe Parents: c5fdacd Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Aug 31 11:47:37 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Wed Aug 31 11:47:42 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/gridtable/GTUtil.java | 53 +++++++++++ .../filter/BuiltInFunctionTupleFilter.java | 22 ++++- .../filter/EvaluatableLikeFunction.java | 96 ++++++++++++++++++++ .../EvaluatableLikeFunctionTransformer.java | 76 ++++++++++++++++ .../kylin/metadata/filter/TupleFilter.java | 2 +- .../metadata/filter/TupleFilterSerializer.java | 3 + .../org/apache/kylin/query/KylinTestBase.java | 8 +- 7 files changed, 254 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java index 4b50176..ce0f016 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java @@ -18,6 +18,7 @@ package org.apache.kylin.gridtable; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.util.Collection; import java.util.List; @@ -25,6 +26,7 @@ import java.util.Set; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; @@ -104,6 +106,13 @@ public class GTUtil { if (encodeConstants && filter instanceof CompareTupleFilter) { return encodeConstants((CompareTupleFilter) filter); } + if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) { + if (!((BuiltInFunctionTupleFilter) filter).hasNested()) { + return encodeConstants((BuiltInFunctionTupleFilter) filter); + } else { + throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down"); + } + } return filter; } @@ -122,9 +131,13 @@ public class GTUtil { return oldCompareFilter; } + //CompareTupleFilter containing BuiltInFunctionTupleFilter will not reach here caz it will be transformed by BuiltInFunctionTransformer CompareTupleFilter newCompareFilter = new CompareTupleFilter(oldCompareFilter.getOperator()); newCompareFilter.addChild(new ColumnTupleFilter(externalCol)); + //for CompareTupleFilter containing dynamicVariables, the below codes will actually replace dynamicVariables + //with normal ConstantTupleFilter + Object firstValue = constValues.iterator().next(); int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol); @@ -199,6 +212,46 @@ public class GTUtil { return result; } + @SuppressWarnings({ "rawtypes", "unchecked" }) + private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) { + // extract ColumnFilter & ConstantFilter + TblColRef externalCol = funcFilter.getColumn(); + + if (externalCol == null) { + return funcFilter; + } + + Collection constValues = funcFilter.getConstantTupleFilter().getValues(); + if (constValues == null || constValues.isEmpty()) { + return funcFilter; + } + + BuiltInFunctionTupleFilter newFuncFilter; + try { + newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName()); + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + throw new RuntimeException(e); + } + newFuncFilter.addChild(new ColumnTupleFilter(externalCol)); + + int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol); + + ByteArray code; + + // translate constant into code + Set newValues = Sets.newHashSet(); + for (Object value : constValues) { + code = translate(col, value, 0); + if (code == null) { + throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter"); + } + newValues.add(code); + } + newFuncFilter.addChild(new ConstantTupleFilter(newValues)); + + return newFuncFilter; + } + transient ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength()); private ByteArray translate(int col, Object value, int roundingFlag) { http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java index 5a10371..40afb18 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.List; +import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.filter.function.BuiltInMethod; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; @@ -40,13 +41,18 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter { protected String name; // FIXME Only supports single parameter functions currently protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower())) + protected ConstantTupleFilter constantTupleFilter; protected int colPosition; protected Method method; protected List<Serializable> methodParams; protected boolean isValidFunc = false; public BuiltInFunctionTupleFilter(String name) { - super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION); + this(name, null); + } + + public BuiltInFunctionTupleFilter(String name, FilterOperatorEnum filterOperatorEnum) { + super(Lists.<TupleFilter> newArrayList(), filterOperatorEnum == null ? FilterOperatorEnum.FUNCTION : filterOperatorEnum); this.methodParams = Lists.newArrayList(); if (name != null) { @@ -59,6 +65,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter { return name; } + public ConstantTupleFilter getConstantTupleFilter() { + return constantTupleFilter; + } + public TblColRef getColumn() { if (columnContainerFilter == null) return null; @@ -71,6 +81,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter { throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter."); } + public boolean hasNested() { + return (columnContainerFilter != null && columnContainerFilter instanceof BuiltInFunctionTupleFilter); + } + public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException { if (columnContainerFilter instanceof ColumnTupleFilter) methodParams.set(colPosition, (Serializable) input); @@ -90,6 +104,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter { colPosition = methodParams.size(); methodParams.add(null); } else if (child instanceof ConstantTupleFilter) { + this.constantTupleFilter = (ConstantTupleFilter) child; Serializable constVal = (Serializable) child.getValues().iterator().next(); try { Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]); @@ -122,12 +137,13 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter { @Override public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { - throw new UnsupportedOperationException("Function filter cannot serialized"); + BytesUtil.writeUTFString(name, buffer); } @Override public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { - throw new UnsupportedOperationException("Function filter cannot serialized"); + this.name = BytesUtil.readUTFString(buffer); + this.initMethod(); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java new file mode 100644 index 0000000..59961ac --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.datatype.StringSerializer; +import org.apache.kylin.metadata.tuple.IEvaluatableTuple; + +public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter { + + public EvaluatableLikeFunction(String name) { + super(name, FilterOperatorEnum.LIKE); + } + + @Override + public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) { + + // extract tuple value + Object tupleValue = null; + for (TupleFilter filter : this.children) { + if (!isConstant(filter)) { + filter.evaluate(tuple, cs); + tupleValue = filter.getValues().iterator().next(); + } + } + + // consider null case + if (cs.isNull(tupleValue)) { + return false; + } + + ByteArray valueByteArray = (ByteArray) tupleValue; + StringSerializer serializer = new StringSerializer(DataType.getType("string")); + String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length())); + try { + return (Boolean) invokeFunction(value); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @Override + public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { + if (!isValid()) { + throw new IllegalStateException("must be valid"); + } + if (methodParams.size() != 2 || methodParams.get(0) != null || methodParams.get(1) == null) { + throw new IllegalArgumentException("bad methodParams: " + methodParams); + } + BytesUtil.writeUTFString(name, buffer); + } + + @Override + public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { + this.name = BytesUtil.readUTFString(buffer); + this.initMethod(); + } + + @Override + public boolean isEvaluable() { + return true; + } + + private boolean isConstant(TupleFilter filter) { + return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter); + } + + public String getLikePattern() { + ByteArray byteArray = (ByteArray) methodParams.get(1); + StringSerializer s = new StringSerializer(DataType.getType("string")); + String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length())); + return pattern; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java new file mode 100644 index 0000000..7295533 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +import java.util.ListIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class EvaluatableLikeFunctionTransformer { + public static final Logger logger = LoggerFactory.getLogger(EvaluatableLikeFunctionTransformer.class); + + public static TupleFilter transform(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + CompareTupleFilter compTupleFilter = (CompareTupleFilter) tupleFilter; + if (compTupleFilter.getFunction() != null && (compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)) { + throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported :" + ((BuiltInFunctionTupleFilter) compTupleFilter.getFunction()).getName()); + } + } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) { + BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) tupleFilter; + if (isLikeFunction(builtInFunctionTupleFilter)) { + for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) { + if (!(child instanceof ColumnTupleFilter) && !(child instanceof ConstantTupleFilter)) { + throw new IllegalArgumentException("Only simple like clause is supported"); + } + } + + translated = new EvaluatableLikeFunction(builtInFunctionTupleFilter.getName()); + for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) { + translated.addChild(child); + } + + } else { + throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported: " + builtInFunctionTupleFilter.getName()); + } + + } else if (tupleFilter instanceof LogicalTupleFilter) { + @SuppressWarnings("unchecked") + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter transformed = transform(childIterator.next()); + if (transformed != null) { + childIterator.set(transformed); + } else { + throw new IllegalStateException("Should not be null"); + } + } + } + return translated == null ? tupleFilter : translated; + } + + private static boolean isLikeFunction(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) { + return "like".equalsIgnoreCase(builtInFunctionTupleFilter.getName()); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java index 3250640..2fb4e1f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java @@ -38,7 +38,7 @@ import com.google.common.collect.Maps; public abstract class TupleFilter { public enum FilterOperatorEnum { - EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36); + EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), LIKE(37); private final int value; http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java index f6c8bc4..0aca9a1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -188,6 +188,9 @@ public class TupleFilterSerializer { case FUNCTION: filter = new BuiltInFunctionTupleFilter(null); break; + case LIKE: + filter = new EvaluatableLikeFunction(null); + break; case MASSIN: filter = new MassInTupleFilter(); break; http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 4e59815..2ad1105 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -63,6 +63,8 @@ import com.google.common.io.Files; */ public class KylinTestBase { + public static boolean PRINT_RESULT = false; + class ObjectArray { Object[] data; @@ -224,7 +226,8 @@ public class KylinTestBase { if (needSort) { queryTable = new SortedTable(queryTable, columnNames); } - //printResult(queryTable); + if (PRINT_RESULT) + printResult(queryTable); return queryTable; } @@ -280,7 +283,8 @@ public class KylinTestBase { if (needSort) { queryTable = new SortedTable(queryTable, columnNames); } - //printResult(queryTable); + if (PRINT_RESULT) + printResult(queryTable); return queryTable; }
