Repository: kylin Updated Branches: refs/heads/master e68b25796 -> ab6e08351
KYLIN-2191 fix integer encoding and refine UT Signed-off-by: Hongbin Ma <mahong...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab6e0835 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab6e0835 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab6e0835 Branch: refs/heads/master Commit: ab6e083517a16bece87db34d9f7f22a5df0662ee Parents: e68b257 Author: Roger Shi <rogershijich...@hotmail.com> Authored: Tue Nov 15 20:59:15 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Nov 15 21:33:45 2016 +0800 ---------------------------------------------------------------------- .../gridtable/DimEncodingPreserveOrderTest.java | 85 +++++++----- .../apache/kylin/dimension/IntegerDimEnc.java | 2 +- .../kylin/dimension/IntegerDimEncTest.java | 128 +++++++++++-------- 3 files changed, 129 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/ab6e0835/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java index ffd43e5..87d37be 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java @@ -20,22 +20,68 @@ package org.apache.kylin.gridtable; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.List; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.dimension.FixedLenHexDimEnc; -import org.apache.kylin.dimension.OneMoreByteVLongDimEnc; import org.apache.kylin.dimension.IntegerDimEnc; +import org.apache.kylin.dimension.OneMoreByteVLongDimEnc; +import org.junit.BeforeClass; import org.junit.Test; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; public class DimEncodingPreserveOrderTest { + private static List<long[]> successValue; + private static List<long[]> failValue; + + @BeforeClass + public static void initTestValue() { + successValue = new ArrayList<>(); + successValue.add(new long[] { -127, 0, 127 }); + successValue.add(new long[] { -32767, -127, 0, 127, 32767 }); + successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 8388607 }); + successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L }); + successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L }); + successValue.add(new long[] { -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L }); + successValue.add(new long[] { -36028797018963967L, -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L, 36028797018963967L }); + successValue.add(new long[] { // + -9223372036854775807L, // + -36028797018963967L, // + -140737488355327L, // + -549755813887L, // + -2147483647L, // + -8388607, // + -32767, // + -127, // + 0, // + 127, // (2 ^ 7) - 1 + 32767, // (2 ^ 15) - 1 + 8388607, // (2 ^ 23) - 1 + 2147483647L, // (2 ^ 31) - 1 + 549755813887L, // (2 ^ 39) - 1 + 140737488355327L, // (2 ^ 47) - 1 + 36028797018963967L, // (2 ^ 55) - 1 + 9223372036854775807L }); // (2 ^ 63) - 1 + + failValue = new ArrayList<>(); + failValue.add(new long[] { -128, 128 }); + failValue.add(new long[] { -32768, 32768 }); + failValue.add(new long[] { -8388608, 8388608 }); + failValue.add(new long[] { -2147483648L, 2147483648L }); + failValue.add(new long[] { -549755813888L, 549755813888L }); + failValue.add(new long[] { -140737488355328L, 140737488355328L }); + failValue.add(new long[] { -36028797018963968L, 36028797018963968L }); + failValue.add(new long[] { -9223372036854775808L }); + } + @Test public void testOneMoreByteVLongDimEncPreserveOrder() { + // TODO: better test OneMoreByteVLongDimEnc enc = new OneMoreByteVLongDimEnc(2); List<ByteArray> encodedValues = Lists.newArrayList(); encodedValues.add(encode(enc, -32768L)); @@ -52,34 +98,15 @@ public class DimEncodingPreserveOrderTest { @Test public void testVLongDimEncPreserveOrder() { - IntegerDimEnc enc = new IntegerDimEnc(2); - List<ByteArray> encodedValues = Lists.newArrayList(); - encodedValues.add(encode(enc, -32767L)); - encodedValues.add(encode(enc, -10000L)); - encodedValues.add(encode(enc, -100L)); - encodedValues.add(encode(enc, 0L)); - encodedValues.add(encode(enc, 100L)); - encodedValues.add(encode(enc, 10000L)); - encodedValues.add(encode(enc, 32767L)); - encodedValues.add(encode(enc, null)); - - assertTrue(Ordering.from(new DefaultGTComparator()).isOrdered(encodedValues)); - } - - @Test - public void testVLongDimEncPreserveOrder2() { - IntegerDimEnc enc = new IntegerDimEnc(8); - List<ByteArray> encodedValues = Lists.newArrayList(); - encodedValues.add(encode(enc, -Long.MAX_VALUE)); - encodedValues.add(encode(enc, -10000L)); - encodedValues.add(encode(enc, -100L)); - encodedValues.add(encode(enc, 0L)); - encodedValues.add(encode(enc, 100L)); - encodedValues.add(encode(enc, 10000L)); - encodedValues.add(encode(enc, Long.MAX_VALUE)); - encodedValues.add(encode(enc, null)); - - assertTrue(Ordering.from(new DefaultGTComparator()).isOrdered(encodedValues)); + for (int i = 1; i <= successValue.size(); i++) { + IntegerDimEnc enc = new IntegerDimEnc(i); + List<ByteArray> encodedValues = Lists.newArrayList(); + for (long value : successValue.get(i - 1)) { + encodedValues.add(encode(enc, value)); + } + encodedValues.add(encode(enc, null)); + assertTrue(Ordering.from(new DefaultGTComparator()).isOrdered(encodedValues)); + } } private ByteArray encode(DimensionEncoding enc, Object value) { http://git-wip-us.apache.org/repos/asf/kylin/blob/ab6e0835/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java index dc9dff7..460ca65 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java @@ -128,7 +128,7 @@ public class IntegerDimEnc extends DimensionEncoding { //only take useful bytes integer = integer & MASK[fixedLen]; - boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0; + boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0; if (!positive) { integer |= (~MASK[fixedLen]); } http://git-wip-us.apache.org/repos/asf/kylin/blob/ab6e0835/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java index 1bdb1d0..a9b2511 100644 --- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java @@ -19,14 +19,60 @@ package org.apache.kylin.dimension; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.metadata.datatype.DataTypeSerializer; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class IntegerDimEncTest { + private static List<long[]> successValue; + private static List<long[]> failValue; + + @BeforeClass + public static void initTestValue() { + successValue = new ArrayList<>(); + successValue.add(new long[] { -127, 0, 127 }); + successValue.add(new long[] { -32767, -127, 0, 127, 32767 }); + successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 8388607 }); + successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L }); + successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L }); + successValue.add(new long[] { -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L }); + successValue.add(new long[] { -36028797018963967L, -140737488355327L, -549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L, 140737488355327L, 36028797018963967L }); + successValue.add(new long[] { // + -9223372036854775807L, // + -36028797018963967L, // + -140737488355327L, // + -549755813887L, // + -2147483647L, // + -8388607, // + -32767, // + -127, // + 0, // + 127, // (2 ^ 7) - 1 + 32767, // (2 ^ 15) - 1 + 8388607, // (2 ^ 23) - 1 + 2147483647L, // (2 ^ 31) - 1 + 549755813887L, // (2 ^ 39) - 1 + 140737488355327L, // (2 ^ 47) - 1 + 36028797018963967L, // (2 ^ 55) - 1 + 9223372036854775807L }); // (2 ^ 63) - 1 + + failValue = new ArrayList<>(); + failValue.add(new long[] { -128, 128 }); + failValue.add(new long[] { -32768, 32768 }); + failValue.add(new long[] { -8388608, 8388608 }); + failValue.add(new long[] { -2147483648L, 2147483648L }); + failValue.add(new long[] { -549755813888L, 549755813888L }); + failValue.add(new long[] { -140737488355328L, 140737488355328L }); + failValue.add(new long[] { -36028797018963968L, 36028797018963968L }); + failValue.add(new long[] { -9223372036854775808L }); + } + @Test public void testConstructor() { try { @@ -66,43 +112,20 @@ public class IntegerDimEncTest { @Test public void testEncodeDecode() { - IntegerDimEnc enc = new IntegerDimEnc(2); - testEncodeDecode(enc, 0); - testEncodeDecode(enc, 100); - testEncodeDecode(enc, 10000); - testEncodeDecode(enc, 32767); - testEncodeDecode(enc, -100); - testEncodeDecode(enc, -10000); - testEncodeDecode(enc, -32767); - try { - testEncodeDecode(enc, 32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage()); - } - try { - testEncodeDecode(enc, -32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage()); - } - } + for (int i = 1; i <= successValue.size(); i++) { + IntegerDimEnc enc = new IntegerDimEnc(i); + for (long value : successValue.get(i - 1)) { + testEncodeDecode(enc, value); + } - @Test - public void testEncodeDecode2() { - IntegerDimEnc enc = new IntegerDimEnc(8); - testEncodeDecode(enc, 0); - testEncodeDecode(enc, 100); - testEncodeDecode(enc, 10000); - testEncodeDecode(enc, Long.MAX_VALUE); - testEncodeDecode(enc, -100); - testEncodeDecode(enc, -10000); - testEncodeDecode(enc, -Long.MAX_VALUE); - try { - testEncodeDecode(enc, Long.MIN_VALUE); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<-9223372036854775808> but was:<null>", e.getMessage()); + for (long value : failValue.get(i - 1)) { + try { + testEncodeDecode(enc, value); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<" + value + "> but was:<null>", e.getMessage()); + } + } } } @@ -117,25 +140,19 @@ public class IntegerDimEncTest { @Test public void testSerDes() { - IntegerDimEnc enc = new IntegerDimEnc(2); - testSerDes(enc, 0); - testSerDes(enc, 100); - testSerDes(enc, 10000); - testSerDes(enc, 32767); - testSerDes(enc, -100); - testSerDes(enc, -10000); - testSerDes(enc, -32767); - try { - testSerDes(enc, 32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage()); - } - try { - testSerDes(enc, -32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage()); + for (int i = 1; i <= successValue.size(); i++) { + IntegerDimEnc enc = new IntegerDimEnc(i); + for (long value : successValue.get(i - 1)) { + testSerDes(enc, value); + } + for (long value : failValue.get(i - 1)) { + try { + testSerDes(enc, value); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<" + value + "> but was:<null>", e.getMessage()); + } + } } } @@ -147,5 +164,4 @@ public class IntegerDimEncTest { String decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); Assert.assertEquals(valueStr, decode); } - }
