This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch sync in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 959b74777392a34588cc46c2795c0c9e971462e9 Author: Li Yang <[email protected]> AuthorDate: Wed May 23 20:58:47 2018 +0800 KYLIN-3370 refactor DataTypeOrder --- .../apache/kylin/cube/kv/RowKeyColumnOrder.java | 108 -------- .../apache/kylin/metadata/datatype/DataType.java | 26 +- .../kylin/metadata/datatype/DataTypeOrder.java | 155 ++++++++++++ .../kylin/storage/translate/ColumnValueRange.java | 214 ---------------- .../storage/translate/DerivedFilterTranslator.java | 6 +- .../kylin/storage/translate/HBaseKeyRange.java | 273 --------------------- .../storage/translate/ColumnValueRangeTest.java | 126 ---------- 7 files changed, 167 insertions(+), 741 deletions(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java deleted file mode 100644 index 3f4f6f4..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.kv; - -import java.util.Collection; -import java.util.Comparator; - -import org.apache.kylin.metadata.datatype.DataType; - -/** - * @author yangli9 - */ -abstract public class RowKeyColumnOrder implements Comparator<String> { - - public static final NumberOrder NUMBER_ORDER = new NumberOrder(); - public static final StringOrder STRING_ORDER = new StringOrder(); - - public static RowKeyColumnOrder getInstance(DataType type) { - if (type.isNumberFamily() || type.isDateTimeFamily()) - return NUMBER_ORDER; - else - return STRING_ORDER; - } - - public String max(Collection<String> values) { - String max = null; - for (String v : values) { - if (max == null || compare(max, v) < 0) - max = v; - } - return max; - } - - public String min(Collection<String> values) { - String min = null; - for (String v : values) { - if (min == null || compare(min, v) > 0) - min = v; - } - return min; - } - - public String min(String v1, String v2) { - if (v1 == null) - return v2; - else if (v2 == null) - return v1; - else - return compare(v1, v2) <= 0 ? v1 : v2; - } - - public String max(String v1, String v2) { - if (v1 == null) - return v2; - else if (v2 == null) - return v1; - else - return compare(v1, v2) >= 0 ? v1 : v2; - } - - @Override - public int compare(String o1, String o2) { - // consider null - if (o1 == o2) - return 0; - if (o1 == null) - return -1; - if (o2 == null) - return 1; - - return compareNonNull(o1, o2); - } - - abstract int compareNonNull(String o1, String o2); - - private static class StringOrder extends RowKeyColumnOrder { - @Override - public int compareNonNull(String o1, String o2) { - return o1.compareTo(o2); - } - } - - private static class NumberOrder extends RowKeyColumnOrder { - @Override - public int compareNonNull(String o1, String o2) { - double d1 = Double.parseDouble(o1); - double d2 = Double.parseDouble(o2); - return Double.compare(d1, d2); - } - } - -} diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java index 5ccc1f3..d9eefb5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java @@ -19,7 +19,6 @@ package org.apache.kylin.metadata.datatype; import java.io.Serializable; -import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.HashSet; @@ -35,7 +34,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BytesSerializer; import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.measure.MeasureTypeFactory; import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum; @@ -162,6 +160,7 @@ public class DataType implements Serializable { private String name; private int precision; private int scale; + private transient DataTypeOrder order; public DataType(String name, int precision, int scale) { this.name = name; @@ -224,24 +223,17 @@ public class DataType implements Serializable { scale = KylinConfig.getInstanceFromEnv().getDefaultDecimalScale(); } } - } + public DataTypeOrder getOrder() { + if (order == null) + order = DataTypeOrder.getInstance(this); + + return order; + } + public int compare(String value1, String value2) { - if (isDateTimeFamily()) { - Long millis1 = DateFormat.stringToMillis(value1); - Long millis2 = DateFormat.stringToMillis(value2); - return millis1.compareTo(millis2); - } else if (isIntegerFamily()) { - Long l1 = new Long(value1); - Long l2 = new Long(value2); - return l1.compareTo(l2); - } else if (isNumberFamily()) { - BigDecimal bigDecimal1 = new BigDecimal(value1); - BigDecimal bigDecimal2 = new BigDecimal(value2); - return bigDecimal1.compareTo(bigDecimal2); - } - return value1.compareTo(value2); + return getOrder().compare(value1, value2); } private String replaceLegacy(String str) { diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeOrder.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeOrder.java new file mode 100644 index 0000000..091e2ae --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeOrder.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.metadata.datatype; + +import java.math.BigDecimal; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.kylin.common.util.DateFormat; + +/** + * Define order for string literals based on its underlying data type. + * + * Null is the smallest. + */ +abstract public class DataTypeOrder implements Comparator<String> { + + public static final DataTypeOrder INTEGER_ORDER = new IntegerOrder(); + public static final DataTypeOrder DOUBLE_ORDER = new DoubleOrder(); + public static final DataTypeOrder DECIMAL_ORDER = new DecimalOrder(); + public static final DataTypeOrder DATETIME_ORDER = new DateTimeOrder(); + public static final DataTypeOrder STRING_ORDER = new StringOrder(); + + // package private, access via DataType.getOrder() + static DataTypeOrder getInstance(DataType type) throws IllegalArgumentException { + if (type.isStringFamily()) + return STRING_ORDER; + else if (type.isDateTimeFamily()) + return DATETIME_ORDER; + else if (type.isIntegerFamily()) + return INTEGER_ORDER; + else if (type.isFloat() || type.isDouble()) + return DOUBLE_ORDER; + else if (type.isDecimal()) + return DECIMAL_ORDER; + else + throw new IllegalArgumentException("Unsupported data type " + type); + } + + public String max(Collection<String> values) { + String max = null; + for (String v : values) { + max = max(max, v); + } + return max; + } + + public String min(Collection<String> values) { + String min = null; + for (String v : values) { + min = min(min, v); + } + return min; + } + + public String min(String v1, String v2) { + if (v1 == null) + return v2; + else if (v2 == null) + return v1; + else + return compare(v1, v2) <= 0 ? v1 : v2; + } + + public String max(String v1, String v2) { + if (v1 == null) + return v2; + else if (v2 == null) + return v1; + else + return compare(v1, v2) >= 0 ? v1 : v2; + } + + @Override + public int compare(String s1, String s2) { + Comparable o1 = toComparable(s1); + Comparable o2 = toComparable(s2); + + // consider null + if (o1 == o2) + return 0; + if (o1 == null) + return -1; + if (o2 == null) + return 1; + + return o1.compareTo(o2); + } + + abstract Comparable toComparable(String s); + + private static class StringOrder extends DataTypeOrder { + @Override + public String toComparable(String s) { + return s; + } + } + + private static class IntegerOrder extends DataTypeOrder { + @Override + public Long toComparable(String s) { + if (s == null || s.isEmpty()) + return null; + else + return Long.parseLong(s); + } + } + + private static class DoubleOrder extends DataTypeOrder { + @Override + public Double toComparable(String s) { + if (s == null || s.isEmpty()) + return null; + else + return Double.parseDouble(s); + } + } + + private static class DecimalOrder extends DataTypeOrder { + @Override + public BigDecimal toComparable(String s) { + if (s == null || s.isEmpty()) + return null; + else + return new BigDecimal(s); + } + } + + private static class DateTimeOrder extends DataTypeOrder { + @Override + public Long toComparable(String s) { + if (s == null || s.isEmpty()) + return null; + else + return DateFormat.stringToMillis(s); + } + } + +} diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java deleted file mode 100644 index 56b1106..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.translate; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.cube.kv.RowKeyColumnOrder; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; - -import com.google.common.collect.Sets; - -/** - * - * @author xjiang - * - */ -public class ColumnValueRange { - private TblColRef column; - private RowKeyColumnOrder order; - private String beginValue; - private String endValue; - private Set<String> equalValues; - - public ColumnValueRange(TblColRef column, Collection<String> values, FilterOperatorEnum op) { - this.column = column; - this.order = RowKeyColumnOrder.getInstance(column.getType()); - - switch (op) { - case EQ: - case IN: - equalValues = new HashSet<String>(values); - refreshBeginEndFromEquals(); - break; - case LT: - case LTE: - endValue = order.max(values); - break; - case GT: - case GTE: - beginValue = order.min(values); - break; - case NEQ: - case NOTIN: - case ISNULL: // TODO ISNULL worth pass down as a special equal value - case ISNOTNULL: - // let Optiq filter it! - break; - default: - throw new UnsupportedOperationException(op.name()); - } - } - - public ColumnValueRange(TblColRef column, String beginValue, String endValue, Set<String> equalValues) { - copy(column, beginValue, endValue, equalValues); - } - - void copy(TblColRef column, String beginValue, String endValue, Set<String> equalValues) { - this.column = column; - this.order = RowKeyColumnOrder.getInstance(column.getType()); - this.beginValue = beginValue; - this.endValue = endValue; - this.equalValues = equalValues; - } - - public TblColRef getColumn() { - return column; - } - - public String getBeginValue() { - return beginValue; - } - - public String getEndValue() { - return endValue; - } - - public Set<String> getEqualValues() { - return equalValues; - } - - private void refreshBeginEndFromEquals() { - this.beginValue = order.min(this.equalValues); - this.endValue = order.max(this.equalValues); - } - - public boolean satisfyAll() { - return beginValue == null && endValue == null && equalValues == null; // the NEQ case - } - - public boolean satisfyNone() { - if (equalValues != null) { - return equalValues.isEmpty(); - } else if (beginValue != null && endValue != null) { - return order.compare(beginValue, endValue) > 0; - } else { - return false; - } - } - - public void andMerge(ColumnValueRange another) { - assert this.column.equals(another.column); - - if (another.satisfyAll()) { - return; - } - - if (this.satisfyAll()) { - copy(another.column, another.beginValue, another.endValue, another.equalValues); - return; - } - - if (this.equalValues != null && another.equalValues != null) { - this.equalValues.retainAll(another.equalValues); - refreshBeginEndFromEquals(); - return; - } - - if (this.equalValues != null) { - this.equalValues = filter(this.equalValues, another.beginValue, another.endValue); - refreshBeginEndFromEquals(); - return; - } - - if (another.equalValues != null) { - this.equalValues = filter(another.equalValues, this.beginValue, this.endValue); - refreshBeginEndFromEquals(); - return; - } - - this.beginValue = order.max(this.beginValue, another.beginValue); - this.endValue = order.min(this.endValue, another.endValue); - } - - private Set<String> filter(Set<String> equalValues, String beginValue, String endValue) { - Set<String> result = Sets.newHashSetWithExpectedSize(equalValues.size()); - for (String v : equalValues) { - if (between(v, beginValue, endValue)) { - result.add(v); - } - } - return equalValues; - } - - private boolean between(String v, String beginValue, String endValue) { - return (beginValue == null || order.compare(beginValue, v) <= 0) && (endValue == null || order.compare(v, endValue) <= 0); - } - - // remove invalid EQ/IN values and round start/end according to dictionary - public void preEvaluateWithDict(Dictionary<String> dict) { - if (dict == null || dict.getSize() == 0) - return; - - if (equalValues != null) { - Iterator<String> it = equalValues.iterator(); - while (it.hasNext()) { - String v = it.next(); - try { - dict.getIdFromValue(v); - } catch (IllegalArgumentException e) { - // value not in dictionary - it.remove(); - } - } - refreshBeginEndFromEquals(); - } - - if (beginValue != null) { - try { - beginValue = dict.getValueFromId(dict.getIdFromValue(beginValue, 1)); - } catch (IllegalArgumentException e) { - // beginValue is greater than the biggest in dictionary, mark FALSE - equalValues = Sets.newHashSet(); - } - } - - if (endValue != null) { - try { - endValue = dict.getValueFromId(dict.getIdFromValue(endValue, -1)); - } catch (IllegalArgumentException e) { - // endValue is lesser than the smallest in dictionary, mark FALSE - equalValues = Sets.newHashSet(); - } - } - } - - public String toString() { - if (equalValues == null) { - return column.getName() + " between " + beginValue + " and " + endValue; - } else { - return column.getName() + " in " + equalValues; - } - } -} diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java index 7fa426f..4a80d29 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java @@ -24,10 +24,10 @@ import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.Pair; -import org.apache.kylin.cube.kv.RowKeyColumnOrder; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.cube.model.CubeDesc.DeriveType; import org.apache.kylin.dict.lookup.ILookupTable; +import org.apache.kylin.metadata.datatype.DataTypeOrder; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; @@ -164,9 +164,9 @@ public class DerivedFilterTranslator { private static void findMinMax(Set<Array<String>> satisfyingHostRecords, TblColRef[] hostCols, String[] min, String[] max) { - RowKeyColumnOrder[] orders = new RowKeyColumnOrder[hostCols.length]; + DataTypeOrder[] orders = new DataTypeOrder[hostCols.length]; for (int i = 0; i < hostCols.length; i++) { - orders[i] = RowKeyColumnOrder.getInstance(hostCols[i].getType()); + orders[i] = hostCols[i].getType().getOrder(); } for (Array<String> rec : satisfyingHostRecords) { diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java deleted file mode 100644 index 85678ac..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.translate; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.kylin.common.debug.BackdoorToggles; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.cuboid.Cuboid; -import org.apache.kylin.cube.kv.AbstractRowKeyEncoder; -import org.apache.kylin.cube.kv.FuzzyKeyEncoder; -import org.apache.kylin.cube.kv.FuzzyMaskEncoder; -import org.apache.kylin.cube.kv.LazyRowKeyEncoder; -import org.apache.kylin.cube.kv.RowConstants; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -/** - * @author xjiang - */ -public class HBaseKeyRange implements Comparable<HBaseKeyRange> { - - private static final Logger logger = LoggerFactory.getLogger(HBaseKeyRange.class); - - private static final int FUZZY_VALUE_CAP = 100; - private static final byte[] ZERO_TAIL_BYTES = new byte[] { 0 }; - - private final CubeSegment cubeSeg; - private final Cuboid cuboid; - private final List<Collection<ColumnValueRange>> flatOrAndFilter; // OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR .. - - private byte[] startKey; - private byte[] stopKey; - private List<Pair<byte[], byte[]>> fuzzyKeys; - - private String startKeyString; - private String stopKeyString; - private String fuzzyKeyString; - - private long partitionColumnStartDate = Long.MIN_VALUE; - private long partitionColumnEndDate = Long.MAX_VALUE; - - public HBaseKeyRange(CubeSegment cubeSeg, Cuboid cuboid, byte[] startKey, byte[] stopKey, List<Pair<byte[], byte[]>> fuzzyKeys, List<Collection<ColumnValueRange>> flatColumnValueFilter, long partitionColumnStartDate, long partitionColumnEndDate) { - this.cubeSeg = cubeSeg; - this.cuboid = cuboid; - this.startKey = startKey; - this.stopKey = stopKey; - this.fuzzyKeys = fuzzyKeys; - this.flatOrAndFilter = flatColumnValueFilter; - this.partitionColumnStartDate = partitionColumnStartDate; - this.partitionColumnEndDate = partitionColumnEndDate; - initDebugString(); - } - - public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) { - this.cubeSeg = cubeSeg; - long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns); - this.cuboid = Cuboid.findById(cubeSeg, cuboidId); - this.flatOrAndFilter = Lists.newLinkedList(); - this.flatOrAndFilter.add(andDimensionRanges); - init(andDimensionRanges); - initDebugString(); - } - - private long calculateCuboidID(CubeDesc cube, Collection<TblColRef> dimensions) { - long cuboidID = 0; - for (TblColRef column : dimensions) { - int index = cube.getRowkey().getColumnBitIndex(column); - cuboidID |= 1L << index; - } - return cuboidID; - } - - private void init(Collection<ColumnValueRange> andDimensionRanges) { - int size = andDimensionRanges.size(); - Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size); - Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size); - Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size); - for (ColumnValueRange dimRange : andDimensionRanges) { - TblColRef column = dimRange.getColumn(); - startValues.put(column, dimRange.getBeginValue()); - stopValues.put(column, dimRange.getEndValue()); - fuzzyValues.put(column, dimRange.getEqualValues()); - - TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef(); - if (column.equals(partitionDateColumnRef)) { - initPartitionRange(dimRange); - } - } - - AbstractRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid); - encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE); - this.startKey = encoder.encode(startValues); - encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE); - // In order to make stopRow inclusive add a trailing 0 byte. #See Scan.setStopRow(byte [] stopRow) - this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES); - - // always fuzzy match cuboid ID to lock on the selected cuboid - this.fuzzyKeys = buildFuzzyKeys(fuzzyValues); - } - - private void initPartitionRange(ColumnValueRange dimRange) { - if (null != dimRange.getBeginValue()) { - this.partitionColumnStartDate = DateFormat.stringToMillis(dimRange.getBeginValue()); - } - if (null != dimRange.getEndValue()) { - this.partitionColumnEndDate = DateFormat.stringToMillis(dimRange.getEndValue()); - } - } - - private void initDebugString() { - this.startKeyString = BytesUtil.toHex(this.startKey); - this.stopKeyString = BytesUtil.toHex(this.stopKey); - StringBuilder buf = new StringBuilder(); - for (Pair<byte[], byte[]> fuzzyKey : this.fuzzyKeys) { - buf.append(BytesUtil.toHex(fuzzyKey.getFirst())); - buf.append(" "); - buf.append(BytesUtil.toHex(fuzzyKey.getSecond())); - buf.append(";"); - } - this.fuzzyKeyString = buf.toString(); - } - - private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) { - ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>(); - - // debug/profiling purpose - if (BackdoorToggles.getDisableFuzzyKey()) { - logger.info("The execution of this query will not use fuzzy key"); - return result; - } - - FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid); - FuzzyMaskEncoder fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid); - - List<Map<TblColRef, String>> fuzzyValues = FuzzyValueCombination.calculate(fuzzyValueSet, FUZZY_VALUE_CAP); - for (Map<TblColRef, String> fuzzyValue : fuzzyValues) { - result.add(Pair.newPair(fuzzyKeyEncoder.encode(fuzzyValue), fuzzyMaskEncoder.encode(fuzzyValue))); - } - return result; - } - - public CubeSegment getCubeSegment() { - return this.cubeSeg; - } - - public Cuboid getCuboid() { - return cuboid; - } - - public byte[] getStartKey() { - return startKey; - } - - public byte[] getStopKey() { - return stopKey; - } - - public List<Pair<byte[], byte[]>> getFuzzyKeys() { - return fuzzyKeys; - } - - public String getStartKeyAsString() { - return startKeyString; - } - - public String getStopKeyAsString() { - return stopKeyString; - } - - public String getFuzzyKeyAsString() { - return fuzzyKeyString; - } - - public List<Collection<ColumnValueRange>> getFlatOrAndFilter() { - return flatOrAndFilter; - } - - public long getPartitionColumnStartDate() { - return partitionColumnStartDate; - } - - public long getPartitionColumnEndDate() { - return partitionColumnEndDate; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((cubeSeg == null) ? 0 : cubeSeg.hashCode()); - result = prime * result + ((cuboid == null) ? 0 : cuboid.hashCode()); - result = prime * result + ((fuzzyKeyString == null) ? 0 : fuzzyKeyString.hashCode()); - result = prime * result + ((startKeyString == null) ? 0 : startKeyString.hashCode()); - result = prime * result + ((stopKeyString == null) ? 0 : stopKeyString.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - HBaseKeyRange other = (HBaseKeyRange) obj; - if (cubeSeg == null) { - if (other.cubeSeg != null) - return false; - } else if (!cubeSeg.equals(other.cubeSeg)) - return false; - if (cuboid == null) { - if (other.cuboid != null) - return false; - } else if (!cuboid.equals(other.cuboid)) - return false; - if (fuzzyKeyString == null) { - if (other.fuzzyKeyString != null) - return false; - } else if (!fuzzyKeyString.equals(other.fuzzyKeyString)) - return false; - if (startKeyString == null) { - if (other.startKeyString != null) - return false; - } else if (!startKeyString.equals(other.startKeyString)) - return false; - if (stopKeyString == null) { - if (other.stopKeyString != null) - return false; - } else if (!stopKeyString.equals(other.stopKeyString)) - return false; - return true; - } - - @Override - public int compareTo(HBaseKeyRange other) { - return Bytes.compareTo(this.startKey, other.startKey); - } - - public boolean hitSegment() { - return cubeSeg.getTSRange().start.v <= getPartitionColumnEndDate() && cubeSeg.getTSRange().end.v >= getPartitionColumnStartDate(); - } -} diff --git a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java deleted file mode 100644 index 0e7e91f..0000000 --- a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.translate; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.dict.StringBytesConverter; -import org.apache.kylin.dict.TrieDictionaryBuilder; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class ColumnValueRangeTest extends LocalFileMetadataTestCase { - - @BeforeClass - public static void setUp() throws Exception { - staticCreateTestMetadata(); - } - - @AfterClass - public static void after() throws Exception { - cleanAfterClass(); - } - - @Test - public void testPreEvaluateWithDict() { - TblColRef col = mockupTblColRef(); - Dictionary<String> dict = mockupDictionary(col, "CN", "US"); - - ColumnValueRange r1 = new ColumnValueRange(col, set("CN", "US", "Other"), FilterOperatorEnum.EQ); - r1.preEvaluateWithDict(dict); - assertEquals(set("CN", "US"), r1.getEqualValues()); - - // less than rounding - { - ColumnValueRange r2 = new ColumnValueRange(col, set("CN"), FilterOperatorEnum.LT); - r2.preEvaluateWithDict(dict); - assertEquals(null, r2.getBeginValue()); - assertEquals("CN", r2.getEndValue()); - - ColumnValueRange r3 = new ColumnValueRange(col, set("Other"), FilterOperatorEnum.LT); - r3.preEvaluateWithDict(dict); - assertEquals(null, r3.getBeginValue()); - assertEquals("CN", r3.getEndValue()); - - ColumnValueRange r4 = new ColumnValueRange(col, set("UT"), FilterOperatorEnum.LT); - r4.preEvaluateWithDict(dict); - assertEquals(null, r4.getBeginValue()); - assertEquals("US", r4.getEndValue()); - } - - // greater than rounding - { - ColumnValueRange r2 = new ColumnValueRange(col, set("CN"), FilterOperatorEnum.GTE); - r2.preEvaluateWithDict(dict); - assertEquals("CN", r2.getBeginValue()); - assertEquals(null, r2.getEndValue()); - - ColumnValueRange r3 = new ColumnValueRange(col, set("Other"), FilterOperatorEnum.GTE); - r3.preEvaluateWithDict(dict); - assertEquals("US", r3.getBeginValue()); - assertEquals(null, r3.getEndValue()); - - ColumnValueRange r4 = new ColumnValueRange(col, set("CI"), FilterOperatorEnum.GTE); - r4.preEvaluateWithDict(dict); - assertEquals("CN", r4.getBeginValue()); - assertEquals(null, r4.getEndValue()); - } - - // ever false check - { - ColumnValueRange r2 = new ColumnValueRange(col, set("UT"), FilterOperatorEnum.GTE); - r2.preEvaluateWithDict(dict); - assertTrue(r2.satisfyNone()); - - ColumnValueRange r3 = new ColumnValueRange(col, set("CM"), FilterOperatorEnum.LT); - r3.preEvaluateWithDict(dict); - assertTrue(r3.satisfyNone()); - } - } - - public static Dictionary<String> mockupDictionary(TblColRef col, String... values) { - TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<String>(new StringBytesConverter()); - for (String v : values) { - builder.addValue(v); - } - return builder.build(0); - } - - private static Set<String> set(String... values) { - HashSet<String> list = new HashSet<String>(); - list.addAll(Arrays.asList(values)); - return list; - } - - public static TblColRef mockupTblColRef() { - TableDesc t = TableDesc.mockup("table_a"); - return TblColRef.mockup(t, 1, "col_1", "string"); - } -}
