refactor: remove maskForEqualHashComp from GTRecord
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64b5f86f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64b5f86f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64b5f86f Branch: refs/heads/master Commit: 64b5f86fec6fce643cbd0f0f5a966ddc919dfbed Parents: ff1c813 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Jun 21 14:50:27 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Wed Jun 22 14:00:15 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BasicTest.java | 3 + .../kylin/gridtable/FuzzyKeyGTRecord.java | 37 ++++++++++++ .../org/apache/kylin/gridtable/GTRecord.java | 60 ++++++-------------- .../org/apache/kylin/gridtable/GTScanRange.java | 16 +++--- .../kylin/gridtable/GTScanRangePlanner.java | 16 +++--- .../apache/kylin/gridtable/GTScanRequest.java | 21 +++++-- .../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 ++- 7 files changed, 91 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index f1f5aa4..81afafe 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -79,6 +79,9 @@ public class BasicTest { @Test public void testxx() throws InterruptedException { + byte[] space = new byte[100]; + ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20); + buffer.put((byte) 1); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java new file mode 100644 index 0000000..acffb4d --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.gridtable; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.ImmutableBitSet; + +public class FuzzyKeyGTRecord extends GTRecord { + final ImmutableBitSet maskForEqualHashComp; + + public FuzzyKeyGTRecord(GTInfo info, ByteArray[] cols, ImmutableBitSet maskForEqualHashComp) { + super(info, cols); + this.maskForEqualHashComp = maskForEqualHashComp; + } + + public FuzzyKeyGTRecord(GTInfo info,ImmutableBitSet maskForEqualHashComp) + { + super(info); + this.maskForEqualHashComp = maskForEqualHashComp; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index f3cfc6a..b676693 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -31,39 +31,27 @@ public class GTRecord implements Comparable<GTRecord> { final transient GTInfo info; final ByteArray[] cols; - final ImmutableBitSet maskForEqualHashComp; - public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp, ByteArray[] cols) { + public GTRecord(GTInfo info, ByteArray[] cols) { this.info = info; this.cols = cols; - this.maskForEqualHashComp = maskForEqualHashComp; } - public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp) { + public GTRecord(GTInfo info) { this.cols = new ByteArray[info.getColumnCount()]; for (int i = 0; i < this.cols.length; i++) { // consider column projection by pass in another bit set this.cols[i] = new ByteArray(); } this.info = info; - this.maskForEqualHashComp = maskForEqualHashComp; - } - - public GTRecord(GTInfo info) { - this(info, info.colAll); - } - - public GTRecord(GTInfo info, ByteArray[] cols) { - this(info, info.colAll, cols); } public GTRecord(GTRecord other) { this.info = other.info; - this.maskForEqualHashComp = other.maskForEqualHashComp; this.cols = new ByteArray[info.getColumnCount()]; for (int i = 0; i < other.cols.length; i++) { this.cols[i] = other.cols[i].copy(); - } + } } public GTInfo getInfo() { @@ -124,6 +112,7 @@ public class GTRecord implements Comparable<GTRecord> { return result; } + /** decode and return the values of this record */ public Object[] getValues(int[] selectedColumns, Object[] result) { assert selectedColumns.length <= result.length; for (int i = 0; i < selectedColumns.length; i++) { @@ -136,7 +125,7 @@ public class GTRecord implements Comparable<GTRecord> { } return result; } - + public int sizeOf(ImmutableBitSet selectedCols) { int size = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { @@ -151,15 +140,10 @@ public class GTRecord implements Comparable<GTRecord> { } public GTRecord copy(ImmutableBitSet selectedCols) { - int len = 0; - for (int i = 0; i < selectedCols.trueBitCount(); i++) { - int c = selectedCols.trueBitAt(i); - len += cols[c].length(); - } - + int len = sizeOf(selectedCols); byte[] space = new byte[len]; - GTRecord copy = new GTRecord(info, this.maskForEqualHashComp); + GTRecord copy = new GTRecord(info); int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); @@ -171,10 +155,6 @@ public class GTRecord implements Comparable<GTRecord> { return copy; } - public ImmutableBitSet maskForEqualHashComp() { - return maskForEqualHashComp; - } - @Override public boolean equals(Object obj) { if (this == obj) @@ -187,10 +167,9 @@ public class GTRecord implements Comparable<GTRecord> { GTRecord o = (GTRecord) obj; if (this.info != o.info) return false; - if (this.maskForEqualHashComp != o.maskForEqualHashComp) - return false; - for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) { - int c = maskForEqualHashComp.trueBitAt(i); + + for (int i = 0; i < info.colAll.trueBitCount(); i++) { + int c = info.colAll.trueBitAt(i); if (!this.cols[c].equals(o.cols[c])) { return false; } @@ -201,8 +180,8 @@ public class GTRecord implements Comparable<GTRecord> { @Override public int hashCode() { int hash = 1; - for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) { - int c = maskForEqualHashComp.trueBitAt(i); + for (int i = 0; i < info.colAll.trueBitCount(); i++) { + int c = info.colAll.trueBitAt(i); hash = (31 * hash) + cols[c].hashCode(); } return hash; @@ -210,13 +189,12 @@ public class GTRecord implements Comparable<GTRecord> { @Override public int compareTo(GTRecord o) { - assert this.info == o.info; - assert this.maskForEqualHashComp == o.maskForEqualHashComp; // reference equal for performance + assert this.info == o.info; // reference equal for performance IGTComparator comparator = info.codeSystem.getComparator(); int comp = 0; - for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) { - int c = maskForEqualHashComp.trueBitAt(i); + for (int i = 0; i < info.colAll.trueBitCount(); i++) { + int c = info.colAll.trueBitAt(i); comp = comparator.compare(cols[c], o.cols[c]); if (comp != 0) return comp; @@ -226,7 +204,7 @@ public class GTRecord implements Comparable<GTRecord> { @Override public String toString() { - return toString(maskForEqualHashComp); + return toString(info.colAll); } public String toString(ImmutableBitSet selectedColumns) { @@ -238,11 +216,7 @@ public class GTRecord implements Comparable<GTRecord> { // ============================================================================ public ByteArray exportColumns(ImmutableBitSet selectedCols) { - int len = 0; - for (int i = 0; i < selectedCols.trueBitCount(); i++) { - int c = selectedCols.trueBitAt(i); - len += cols[c].length(); - } + int len = sizeOf(selectedCols); ByteArray buf = ByteArray.allocate(len); exportColumns(selectedCols, buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java index e1b38dc..433626e 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java @@ -27,28 +27,28 @@ public class GTScanRange { final public GTRecord pkStart; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded final public GTRecord pkEnd; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded - final public List<GTRecord> fuzzyKeys; // partial matching primary keys + final public List<FuzzyKeyGTRecord> fuzzyKeys; // partial matching primary keys public GTScanRange(GTRecord pkStart, GTRecord pkEnd) { this(pkStart, pkEnd, null); } - public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys) { + public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys) { GTInfo info = pkStart.info; assert info == pkEnd.info; this.pkStart = pkStart; this.pkEnd = pkEnd; - this.fuzzyKeys = fuzzyKeys == null ? Collections.<GTRecord> emptyList() : fuzzyKeys; + this.fuzzyKeys = fuzzyKeys == null ? Collections.<FuzzyKeyGTRecord> emptyList() : fuzzyKeys; } public GTScanRange replaceGTInfo(final GTInfo gtInfo) { - List<GTRecord> newFuzzyKeys = Lists.newArrayList(); - for (GTRecord input : fuzzyKeys) { - newFuzzyKeys.add(new GTRecord(gtInfo, input.maskForEqualHashComp(), input.cols)); + List<FuzzyKeyGTRecord> newFuzzyKeys = Lists.newArrayList(); + for (FuzzyKeyGTRecord input : fuzzyKeys) { + newFuzzyKeys.add(new FuzzyKeyGTRecord(gtInfo, input.cols, input.maskForEqualHashComp)); } - return new GTScanRange(new GTRecord(gtInfo, pkStart.maskForEqualHashComp(), pkStart.cols), // - new GTRecord(gtInfo, pkEnd.maskForEqualHashComp(), pkEnd.cols), // + return new GTScanRange(new GTRecord(gtInfo, pkStart.cols), // + new GTRecord(gtInfo, pkEnd.cols), // newFuzzyKeys); } http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java index 7d31bf7..2d4c2a2 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java @@ -305,7 +305,7 @@ public class GTScanRangePlanner { GTRecord pkEnd = new GTRecord(gtInfo); Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap(); - List<GTRecord> fuzzyKeys; + List<FuzzyKeyGTRecord> fuzzyKeys; for (ColumnRange range : andDimRanges) { if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) { @@ -337,8 +337,8 @@ public class GTScanRangePlanner { return new GTScanRange(pkStart, pkEnd, fuzzyKeys); } - private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) { - ArrayList<GTRecord> result = Lists.newArrayList(); + private List<FuzzyKeyGTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) { + ArrayList<FuzzyKeyGTRecord> result = Lists.newArrayList(); if (fuzzyValueSet.isEmpty()) return result; @@ -357,7 +357,7 @@ public class GTScanRangePlanner { for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) { bitSet.set(entry.getKey()); } - GTRecord fuzzy = new GTRecord(gtInfo, new ImmutableBitSet(bitSet)); + FuzzyKeyGTRecord fuzzy = new FuzzyKeyGTRecord(gtInfo, new ImmutableBitSet(bitSet)); for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) { fuzzy.set(entry.getKey(), entry.getValue()); } @@ -514,7 +514,7 @@ public class GTScanRangePlanner { GTRecord start = first.pkStart; GTRecord end = first.pkEnd; - List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>(); + List<FuzzyKeyGTRecord> newFuzzyKeys = new ArrayList<FuzzyKeyGTRecord>(); boolean hasNonFuzzyRange = false; for (GTScanRange range : ranges) { @@ -774,12 +774,10 @@ public class GTScanRangePlanner { @Override public int compare(GTRecord a, GTRecord b) { assert a.info == b.info; - assert a.maskForEqualHashComp() == b.maskForEqualHashComp(); - ImmutableBitSet mask = a.maskForEqualHashComp(); int comp; - for (int i = 0; i < mask.trueBitCount(); i++) { - int c = mask.trueBitAt(i); + for (int i = 0; i < a.info.colAll.trueBitCount(); i++) { + int c = a.info.colAll.trueBitAt(i); comp = comparator.compare(a.cols[c], b.cols[c]); if (comp != 0) return comp; http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 0d2da43..66c0e87 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -261,8 +261,8 @@ public class GTScanRequest { serializeGTRecord(range.pkStart, out); serializeGTRecord(range.pkEnd, out); BytesUtil.writeVInt(range.fuzzyKeys.size(), out); - for (GTRecord f : range.fuzzyKeys) { - serializeGTRecord(f, out); + for (FuzzyKeyGTRecord f : range.fuzzyKeys) { + serializeFuzzyKeyGTRecord(f, out); } } @@ -285,10 +285,10 @@ public class GTScanRequest { for (int rangeIdx = 0; rangeIdx < sRangesCount; rangeIdx++) { GTRecord sPkStart = deserializeGTRecord(in, sInfo); GTRecord sPkEnd = deserializeGTRecord(in, sInfo); - List<GTRecord> sFuzzyKeys = Lists.newArrayList(); + List<FuzzyKeyGTRecord> sFuzzyKeys = Lists.newArrayList(); int sFuzzyKeySize = BytesUtil.readVInt(in); for (int i = 0; i < sFuzzyKeySize; i++) { - sFuzzyKeys.add(deserializeGTRecord(in, sInfo)); + sFuzzyKeys.add(deserializeFuzzyKeyGTRecord(in, sInfo)); } GTScanRange sRange = new GTScanRange(sPkStart, sPkEnd, sFuzzyKeys); sRanges.add(sRange); @@ -311,7 +311,6 @@ public class GTScanRequest { for (ByteArray col : gtRecord.cols) { col.exportData(out); } - ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out); } private GTRecord deserializeGTRecord(ByteBuffer in, GTInfo sInfo) { @@ -320,8 +319,18 @@ public class GTScanRequest { for (int i = 0; i < colLength; i++) { sCols[i] = ByteArray.importData(in); } + return new GTRecord(sInfo, sCols); + } + + private void serializeFuzzyKeyGTRecord(FuzzyKeyGTRecord gtRecord, ByteBuffer out) { + serializeGTRecord(gtRecord,out); + ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out); + } + + private FuzzyKeyGTRecord deserializeFuzzyKeyGTRecord(ByteBuffer in, GTInfo sInfo) { + GTRecord temp = deserializeGTRecord(in,sInfo); ImmutableBitSet sMaskForEqualHashComp = ImmutableBitSet.serializer.deserialize(in); - return new GTRecord(sInfo, sMaskForEqualHashComp, sCols); + return new FuzzyKeyGTRecord(temp.info,temp.cols, sMaskForEqualHashComp); } }; http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java index af5d4b7..015edc6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java @@ -40,6 +40,7 @@ import org.apache.kylin.cube.kv.RowKeyEncoder; import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.cube.model.HBaseColumnFamilyDesc; import org.apache.kylin.cube.model.HBaseMappingDesc; +import org.apache.kylin.gridtable.FuzzyKeyGTRecord; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.gridtable.GTScanRange; @@ -92,7 +93,7 @@ public abstract class CubeHBaseRPC implements IGTStorage { return scan; } - private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) { + private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) { final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks); LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid); @@ -136,13 +137,13 @@ public abstract class CubeHBaseRPC implements IGTStorage { * translate GTRecord format fuzzy keys to hbase expected format * @return */ - private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { + private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<FuzzyKeyGTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); - for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { + for (FuzzyKeyGTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf();
