Repository: kylin Updated Branches: refs/heads/master 71f373507 -> ed643e6b2
KYLIN-2055 Add an encoder for Boolean type Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb2b12b3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb2b12b3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb2b12b3 Branch: refs/heads/master Commit: cb2b12b3b619ac86efbb9c7ca708418882683daf Parents: 71f3735 Author: shaofengshi <shaofeng...@apache.org> Authored: Mon Oct 10 13:30:27 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Mon Oct 10 13:30:27 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/dimension/BooleanDimEnc.java | 196 +++++++++++++++++++ .../dimension/DimensionEncodingFactory.java | 1 + .../kylin/dimension/BooleanDimEncTest.java | 95 +++++++++ .../cubeDesigner/advanced_settings.html | 2 +- 4 files changed, 293 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/cb2b12b3/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java new file mode 100644 index 0000000..f32724c --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; + +import com.google.common.collect.Maps; +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; + +/** + * Encoding Boolean values to bytes + */ +public class BooleanDimEnc extends DimensionEncoding { + private static final long serialVersionUID = 1L; + + public static final String ENCODING_NAME = "boolean"; + + //NOTE: when add new value, append to the array tail, DO NOT insert! + public static String[] ALLOWED_VALUES = new String[] { "", "true", "false", "TRUE", "FALSE", "True", "False", "t", "f", "T", "F", "yes", "no", "YES", "NO", "Yes", "No", "y", "n", "Y", "N", "1", "0" }; + + public static final Map<String, Integer> map = Maps.newHashMap(); + + static { + for (int i = 0; i < ALLOWED_VALUES.length; i++) { + map.put(ALLOWED_VALUES[i], i); + } + } + + public static class Factory extends DimensionEncodingFactory { + @Override + public String getSupportedEncodingName() { + return ENCODING_NAME; + } + + @Override + public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { + return new BooleanDimEnc(); + } + }; + + // ============================================================================ + + private static int fixedLen = 1; + + //no-arg constructor is required for Externalizable + public BooleanDimEnc() { + } + + @Override + public int getLengthOfEncoding() { + return fixedLen; + } + + @Override + public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { + if (value == null) { + Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); + return; + } + + encode(Bytes.toString(value, 0, valueLen), output, outputOffset); + } + + void encode(String valueStr, byte[] output, int outputOffset) { + if (valueStr == null) { + Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); + return; + } + + Integer encodeValue = map.get(valueStr); + if (encodeValue == null) { + throw new IllegalArgumentException("Value '" + valueStr + "' is not a recognized boolean value."); + } + + BytesUtil.writeLong(encodeValue, output, outputOffset, fixedLen); + } + + @Override + public String decode(byte[] bytes, int offset, int len) { + if (isNull(bytes, offset, len)) { + return null; + } + + int x = (int) BytesUtil.readLong(bytes, offset, len); + if (x >= ALLOWED_VALUES.length) { + throw new IllegalStateException(); + } + + return ALLOWED_VALUES[x]; + } + + @Override + public DataTypeSerializer<Object> asDataTypeSerializer() { + return new BooleanSerializer(); + } + + private class BooleanSerializer extends DataTypeSerializer<Object> { + // be thread-safe and avoid repeated obj creation + private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); + + private byte[] currentBuf() { + byte[] buf = current.get(); + if (buf == null) { + buf = new byte[fixedLen]; + current.set(buf); + } + return buf; + } + + @Override + public void serialize(Object value, ByteBuffer out) { + byte[] buf = currentBuf(); + String valueStr = value == null ? null : value.toString(); + encode(valueStr, buf, 0); + out.put(buf); + } + + @Override + public Object deserialize(ByteBuffer in) { + byte[] buf = currentBuf(); + in.get(buf); + return decode(buf, 0, buf.length); + } + + @Override + public int peekLength(ByteBuffer in) { + return fixedLen; + } + + @Override + public int maxLength() { + return fixedLen; + } + + @Override + public int getStorageBytesEstimate() { + return fixedLen; + } + + @Override + public Object valueOf(String str) { + return str; + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeShort(fixedLen); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + fixedLen = in.readShort(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + BooleanDimEnc that = (BooleanDimEnc) o; + + return fixedLen == that.fixedLen; + + } + + @Override + public int hashCode() { + return fixedLen; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/cb2b12b3/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java index 27bebd7..aba0c26 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java @@ -79,6 +79,7 @@ public abstract class DimensionEncodingFactory { map.put(FixedLenHexDimEnc.ENCODING_NAME, new FixedLenHexDimEnc.Factory()); map.put(DateDimEnc.ENCODING_NAME, new DateDimEnc.Factory()); map.put(TimeDimEnc.ENCODING_NAME, new TimeDimEnc.Factory()); + map.put(BooleanDimEnc.ENCODING_NAME, new BooleanDimEnc.Factory()); // custom encodings String[] clsNames = KylinConfig.getInstanceFromEnv().getCubeDimensionCustomEncodingFactories(); http://git-wip-us.apache.org/repos/asf/kylin/blob/cb2b12b3/core-metadata/src/test/java/org/apache/kylin/dimension/BooleanDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/BooleanDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/BooleanDimEncTest.java new file mode 100644 index 0000000..c6c1416 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/dimension/BooleanDimEncTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.nio.ByteBuffer; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.junit.Assert; +import org.junit.Test; + +public class BooleanDimEncTest { + + @Test + public void testNull() { + BooleanDimEnc enc = new BooleanDimEnc(); + + byte[] buf = new byte[enc.getLengthOfEncoding()]; + enc.encode(null, 0, buf, 0); + Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length)); + String decode = enc.decode(buf, 0, buf.length); + Assert.assertEquals(null, decode); + + buf = new byte[enc.getLengthOfEncoding()]; + DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); + ser.serialize(null, ByteBuffer.wrap(buf)); + Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length)); + decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); + Assert.assertEquals(null, decode); + } + + @Test + public void testEncodeDecode() { + BooleanDimEnc enc = new BooleanDimEnc(); + + for (String x : BooleanDimEnc.ALLOWED_VALUES) { + testEncodeDecode(enc, x); + } + + try { + testEncodeDecode(enc, "FAlse"); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("Value 'FAlse' is not a recognized boolean value.", e.getMessage()); + } + } + + private void testEncodeDecode(BooleanDimEnc enc, String valueStr) { + byte[] buf = new byte[enc.getLengthOfEncoding()]; + byte[] bytes = Bytes.toBytes(valueStr); + enc.encode(bytes, bytes.length, buf, 0); + String decode = enc.decode(buf, 0, buf.length); + Assert.assertEquals(valueStr, decode); + } + + @Test + public void testSerDes() { + BooleanDimEnc enc = new BooleanDimEnc(); + for (String x : BooleanDimEnc.ALLOWED_VALUES) { + testSerDes(enc, x); + } + + try { + testSerDes(enc, "FAlse"); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("Value 'FAlse' is not a recognized boolean value.", e.getMessage()); + } + } + + private void testSerDes(BooleanDimEnc enc, String valueStr) { + DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); + byte[] buf = new byte[enc.getLengthOfEncoding()]; + ser.serialize(valueStr, ByteBuffer.wrap(buf)); + String decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); + Assert.assertEquals(valueStr, decode); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/cb2b12b3/webapp/app/partials/cubeDesigner/advanced_settings.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html index e8cbf9e..34fd492 100755 --- a/webapp/app/partials/cubeDesigner/advanced_settings.html +++ b/webapp/app/partials/cubeDesigner/advanced_settings.html @@ -244,7 +244,7 @@ <!--Column Length --> <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'" tooltip="rowkey column length.." tooltip-trigger="focus" - ng-disabled="rowkey_column.encoding=='dict'||rowkey_column.encoding=='date'||rowkey_column.encoding=='time'" + ng-disabled="rowkey_column.encoding=='dict'||rowkey_column.encoding=='date'||rowkey_column.encoding=='time'||rowkey_column.encoding=='boolean'" ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column);" ng-model="rowkey_column.valueLength" class="form-control">
