Repository: kylin Updated Branches: refs/heads/master 6707cfa40 -> 6bd239f32
KYLIN-1726 increase OLAPQuery.PROP_SCAN_THRESHOLD to 15001 because streaming table has 15000 rows Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bd239f3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bd239f3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bd239f3 Branch: refs/heads/master Commit: 6bd239f322f60eb23b6bd60be6cb5200036daa38 Parents: 6707cfa Author: Hongbin Ma <mahong...@apache.org> Authored: Sun Sep 18 18:21:17 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Sun Sep 18 18:21:17 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BasicTest.java | 1 + .../kylin/dimension/FixedLenHexDimEnc.java | 195 ++++++++++++++++++ .../org/apache/kylin/dimension/VLongDimEnc.java | 196 +++++++++++++++++++ .../org/apache/kylin/query/KylinTestBase.java | 2 +- 4 files changed, 393 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6bd239f3/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index 81afafe..ee15832 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -79,6 +79,7 @@ public class BasicTest { @Test public void testxx() throws InterruptedException { + System.out.println( 0x8fL); byte[] space = new byte[100]; ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20); buffer.put((byte) 1); http://git-wip-us.apache.org/repos/asf/kylin/blob/6bd239f3/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java new file mode 100644 index 0000000..71fe376 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * used to store hex values like "1A2BFF" + * + * FixedLenHexDimEnc does not support expressing NULL values, nulls will become "FFFFFF" after encode and decode + */ +public class FixedLenHexDimEnc extends DimensionEncoding { + private static final long serialVersionUID = 1L; + + private static Logger logger = LoggerFactory.getLogger(FixedLenHexDimEnc.class); + + // row key fixed length place holder + public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9; + + public static final String ENCODING_NAME = "fixed_length_hex"; + + public static class Factory extends DimensionEncodingFactory { + @Override + public String getSupportedEncodingName() { + return ENCODING_NAME; + } + + @Override + public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { + return new FixedLenHexDimEnc(Integer.parseInt(args[0])); + } + }; + + // ============================================================================ + + private int fixedLen; + private int bytelen; + + transient private int avoidVerbose = 0; + + //no-arg constructor is required for Externalizable + public FixedLenHexDimEnc() { + } + + public FixedLenHexDimEnc(int len) { + this.fixedLen = len; + this.bytelen = (fixedLen + 1) / 2; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + FixedLenHexDimEnc that = (FixedLenHexDimEnc) o; + + return fixedLen == that.fixedLen; + + } + + @Override + public int hashCode() { + return fixedLen; + } + + @Override + public int getLengthOfEncoding() { + return fixedLen; + } + + @Override + public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { + if (value == null) { + Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); + return; + } + + if (valueLen > fixedLen) { + if (avoidVerbose++ % 10000 == 0) { + logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueLen + ", will truncate, value string: " + Bytes.toString(value, 0, valueLen) + " times:" + avoidVerbose); + } + } + + int n = Math.min(valueLen, fixedLen); + System.arraycopy(value, 0, output, outputOffset, n); + + if (n < fixedLen) { + Arrays.fill(output, outputOffset + n, outputOffset + fixedLen, ROWKEY_PLACE_HOLDER_BYTE); + } + } + + @Override + public String decode(byte[] bytes, int offset, int len) { + if (isNull(bytes, offset, len)) { + return null; + } + + while (len > 0 && bytes[offset + len - 1] == ROWKEY_PLACE_HOLDER_BYTE) + len--; + + return Bytes.toString(bytes, offset, len); + } + + @Override + public DataTypeSerializer<Object> asDataTypeSerializer() { + return new FixedLenSerializer(); + } + + public class FixedLenSerializer extends DataTypeSerializer<Object> { + // be thread-safe and avoid repeated obj creation + private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); + + private byte[] currentBuf() { + byte[] buf = current.get(); + if (buf == null) { + buf = new byte[fixedLen]; + current.set(buf); + } + return buf; + } + + @Override + public void serialize(Object value, ByteBuffer out) { + byte[] buf = currentBuf(); + byte[] bytes = value == null ? null : Bytes.toBytes(value.toString()); + encode(bytes, bytes == null ? 0 : bytes.length, buf, 0); + out.put(buf); + } + + @Override + public Object deserialize(ByteBuffer in) { + byte[] buf = currentBuf(); + in.get(buf); + return decode(buf, 0, buf.length); + } + + @Override + public int peekLength(ByteBuffer in) { + return fixedLen; + } + + @Override + public int maxLength() { + return fixedLen; + } + + @Override + public int getStorageBytesEstimate() { + return fixedLen; + } + + @Override + public Object valueOf(String str) { + return str; + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeShort(fixedLen); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + fixedLen = in.readShort(); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/6bd239f3/core-metadata/src/main/java/org/apache/kylin/dimension/VLongDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/VLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/VLongDimEnc.java new file mode 100644 index 0000000..000cd92 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/VLongDimEnc.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VLongDimEnc extends DimensionEncoding { + private static final long serialVersionUID = 1L; + + private static Logger logger = LoggerFactory.getLogger(VLongDimEnc.class); + + private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE }; + private static final long[] TAIL = { 0, 0x80L, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE }; + + public static final String ENCODING_NAME = "vlong"; + + public static class Factory extends DimensionEncodingFactory { + @Override + public String getSupportedEncodingName() { + return ENCODING_NAME; + } + + @Override + public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { + return new VLongDimEnc(Integer.parseInt(args[0])); + } + }; + + // ============================================================================ + + private int fixedLen; + private int byteLen; + + transient private int avoidVerbose = 0; + + //no-arg constructor is required for Externalizable + public VLongDimEnc() { + } + + public VLongDimEnc(int len) { + if (len <= 0 || len >= CAP.length) + throw new IllegalArgumentException(); + + this.fixedLen = len; + this.byteLen = (fixedLen + 1) / 2 + 1;//one additional byte to indicate null + } + + @Override + public int getLengthOfEncoding() { + return byteLen; + } + + @Override + public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { + if (value == null) { + Arrays.fill(output, outputOffset, outputOffset + byteLen, NULL); + return; + } + + encode(Bytes.toString(value, 0, valueLen), output, outputOffset); + } + + void encode(String valueStr, byte[] output, int outputOffset) { + if (valueStr == null) { + Arrays.fill(output, outputOffset, outputOffset + byteLen, NULL); + return; + } + + long integer = Long.parseLong(valueStr); + if (integer > CAP[fixedLen]) { + if (avoidVerbose++ % 10000 == 0) { + logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose); + } + } + + BytesUtil.writeLong(integer, output, outputOffset, fixedLen); + } + + @Override + public String decode(byte[] bytes, int offset, int len) { + if (isNull(bytes, offset, len)) { + return null; + } + + long integer = BytesUtil.readLong(bytes, offset, len); + return String.valueOf(integer); + } + + @Override + public DataTypeSerializer<Object> asDataTypeSerializer() { + return new IntegerSerializer(); + } + + public class IntegerSerializer extends DataTypeSerializer<Object> { + // be thread-safe and avoid repeated obj creation + private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); + + private byte[] currentBuf() { + byte[] buf = current.get(); + if (buf == null) { + buf = new byte[fixedLen]; + current.set(buf); + } + return buf; + } + + @Override + public void serialize(Object value, ByteBuffer out) { + byte[] buf = currentBuf(); + String valueStr = value == null ? null : value.toString(); + encode(valueStr, buf, 0); + out.put(buf); + } + + @Override + public Object deserialize(ByteBuffer in) { + byte[] buf = currentBuf(); + in.get(buf); + return decode(buf, 0, buf.length); + } + + @Override + public int peekLength(ByteBuffer in) { + return fixedLen; + } + + @Override + public int maxLength() { + return fixedLen; + } + + @Override + public int getStorageBytesEstimate() { + return fixedLen; + } + + @Override + public Object valueOf(String str) { + return str; + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeShort(fixedLen); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + fixedLen = in.readShort(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + VLongDimEnc that = (VLongDimEnc) o; + + return fixedLen == that.fixedLen; + + } + + @Override + public int hashCode() { + return fixedLen; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/6bd239f3/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 294750e..d0bcf52 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -586,7 +586,7 @@ public class KylinTestBase { //setup cube conn File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config); Properties props = new Properties(); - props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10001"); + props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "15001"); cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props); //setup h2
