Repository: kylin Updated Branches: refs/heads/yang22-cdh5.7 4b6343413 -> 760de8238 (forced update)
KYLIN-2497 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e477b51f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e477b51f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e477b51f Branch: refs/heads/yang22-cdh5.7 Commit: e477b51fef92454d90f8c411f940d2a7b7327bf6 Parents: f3ad6ca Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Mar 10 23:10:49 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Mar 10 23:10:49 2017 +0800 ---------------------------------------------------------------------- .../mr/steps/FactDistinctColumnsMapperBase.java | 2 +- .../mr/steps/FactDistinctHiveColumnsMapper.java | 14 +- .../engine/mr/steps/SelfDefineSortableKey.java | 108 ++++++-------- .../mr/steps/SelfDefineSortableKeyOld.java | 143 +++++++++++++++++++ .../mr/steps/NumberDictionaryForestTest.java | 18 +-- .../mr/steps/SelfDefineSortableKeyTest.java | 26 ++-- 6 files changed, 220 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e477b51f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java index 74aebb0..4a5b744 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java @@ -54,7 +54,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends Kyli protected IMRTableInputFormat flatTableInputFormat; protected Text outputKey = new Text(); - protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(); + //protected SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(); protected Text outputValue = new Text(); protected int errorRecordCounter = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/e477b51f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java index a5c8fc0..476f26c 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java @@ -30,6 +30,7 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.engine.mr.common.BatchConstants; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; +import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.TblColRef; import com.google.common.collect.Lists; @@ -59,6 +60,8 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap private int partitionColumnIndex = -1; private boolean needFetchPartitionCol = true; + private SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(); + @Override protected void setup(Context context) throws IOException { super.setup(context); @@ -146,9 +149,10 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap keyBuffer.put(Bytes.toBytes(reducerIndex)[3]); keyBuffer.put(Bytes.toBytes(fieldValue)); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); - sortableKey.setText(outputKey); + DataType type = factDictCols.get(i).getType(); + sortableKey.init(outputKey, type); //judge type - sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType()); + context.write(sortableKey, EMPTY_TEXT); } } catch (Exception ex) { @@ -167,8 +171,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap keyBuffer.put(MARK_FOR_PARTITION_COL); keyBuffer.put(Bytes.toBytes(fieldValue)); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); - sortableKey.setText(outputKey); - sortableKey.setTypeId((byte) 0); + sortableKey.init(outputKey, (byte) 0); context.write(sortableKey, EMPTY_TEXT); } } @@ -218,8 +221,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap hllBuf.clear(); hll.writeRegisters(hllBuf); outputValue.set(hllBuf.array(), 0, hllBuf.position()); - sortableKey.setText(outputKey); - sortableKey.setTypeId((byte) 0); + sortableKey.init(outputKey, (byte) 0); context.write(sortableKey, outputValue); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/e477b51f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java index b804eef..acec972 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java @@ -17,73 +17,63 @@ */ package org.apache.kylin.engine.mr.steps; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.kylin.metadata.datatype.DataType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -/** - * Created by xiefan on 16-11-1. - */ +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.kylin.metadata.datatype.DataType; + public class SelfDefineSortableKey implements WritableComparable<SelfDefineSortableKey> { + public enum TypeFlag { - NONE_NUMERIC_TYPE, - INTEGER_FAMILY_TYPE, - DOUBLE_FAMILY_TYPE + NONE_NUMERIC_TYPE, INTEGER_FAMILY_TYPE, DOUBLE_FAMILY_TYPE } private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010) - private Text text; + private Text rawKey; - private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKey.class); + private Object keyInObj; public SelfDefineSortableKey() { } - public SelfDefineSortableKey(byte typeId, Text text) { + public SelfDefineSortableKey(Text key, DataType type) { + init(key, getTypeIdByDatatype(type)); + } + + public void init(Text key, byte typeId) { this.typeId = typeId; - this.text = text; + this.rawKey = key; + if (isNumberFamily()) { + String valueStr = new String(key.getBytes(), 1, key.getLength() - 1); + if (isIntegerFamily()) { + this.keyInObj = Integer.parseInt(valueStr); + } else { + this.keyInObj = Double.parseDouble(valueStr); + } + } else { + this.keyInObj = key; + } + } + + public void init(Text key, DataType type) { + init(key, getTypeIdByDatatype(type)); } @Override public int compareTo(SelfDefineSortableKey o) { - if (!o.isNumberFamily()) { - return this.text.compareTo(o.text); + if (this.typeId != o.typeId) + throw new IllegalStateException("Error. Incompatible types"); + if (!isNumberFamily()) { + return ((Text) this.keyInObj).compareTo(((Text) o.keyInObj)); } else { - byte[] data1 = this.text.getBytes(); - byte[] data2 = o.text.getBytes(); - String str1 = new String(data1, 1, data1.length - 1); - String str2 = new String(data2, 1, data2.length - 1); - if (str1 == null || str1.equals("") || str2 == null || str2.equals("")) { - //should not achieve here - logger.error("none numeric value!"); - return 0; - } - if (o.isIntegerFamily()) { //integer type - try { - Long num1 = Long.parseLong(str1); - Long num2 = Long.parseLong(str2); - return num1.compareTo(num2); - } catch (NumberFormatException e) { - logger.error("NumberFormatException when parse integer family number.str1:" + str1 + " str2:" + str2); - e.printStackTrace(); - return 0; - } - } else { //other numeric type - try { - Double num1 = Double.parseDouble(str1); - Double num2 = Double.parseDouble(str2); - return num1.compareTo(num2); - } catch (NumberFormatException e) { - logger.error("NumberFormatException when parse doul family number.str1:" + str1 + " str2:" + str2); - return 0; - } + if (isIntegerFamily()) { + return Integer.compare((Integer) this.keyInObj, (Integer) o.keyInObj); + } else { + return Double.compare((Double) this.keyInObj, (Double) o.keyInObj); } } } @@ -91,14 +81,15 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta @Override public void write(DataOutput dataOutput) throws IOException { dataOutput.writeByte(typeId); - text.write(dataOutput); + rawKey.write(dataOutput); } @Override public void readFields(DataInput dataInput) throws IOException { this.typeId = dataInput.readByte(); - this.text = new Text(); - text.readFields(dataInput); + Text inputKey = new Text(); + inputKey.readFields(dataInput); + init(inputKey, typeId); } public short getTypeId() { @@ -106,7 +97,7 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta } public Text getText() { - return text; + return rawKey; } public boolean isNumberFamily() { @@ -119,25 +110,18 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal()); } - public boolean isOtherNumericFamily() { - return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal()); - } - - public void setTypeIdByDatatype(DataType type) { + public byte getTypeIdByDatatype(DataType type) { if (!type.isNumberFamily()) { - this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal(); + return (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal(); } else if (type.isIntegerFamily()) { - this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal(); + return (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal(); } else { - this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal(); + return (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal(); } } - + public void setTypeId(byte typeId) { this.typeId = typeId; } - public void setText(Text text) { - this.text = text; - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/e477b51f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java new file mode 100644 index 0000000..aa7fce3 --- /dev/null +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.engine.mr.steps; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.kylin.metadata.datatype.DataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Created by xiefan on 16-11-1. + */ +public class SelfDefineSortableKeyOld implements WritableComparable<SelfDefineSortableKeyOld> { + public enum TypeFlag { + NONE_NUMERIC_TYPE, + INTEGER_FAMILY_TYPE, + DOUBLE_FAMILY_TYPE + } + + private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010) + + private Text text; + + private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKeyOld.class); + + public SelfDefineSortableKeyOld() { + } + + public SelfDefineSortableKeyOld(byte typeId, Text text) { + this.typeId = typeId; + this.text = text; + } + + @Override + public int compareTo(SelfDefineSortableKeyOld o) { + if (!o.isNumberFamily()) { + return this.text.compareTo(o.text); + } else { + byte[] data1 = this.text.getBytes(); + byte[] data2 = o.text.getBytes(); + String str1 = new String(data1, 1, data1.length - 1); + String str2 = new String(data2, 1, data2.length - 1); + if (str1 == null || str1.equals("") || str2 == null || str2.equals("")) { + //should not achieve here + logger.error("none numeric value!"); + return 0; + } + if (o.isIntegerFamily()) { //integer type + try { + Long num1 = Long.parseLong(str1); + Long num2 = Long.parseLong(str2); + return num1.compareTo(num2); + } catch (NumberFormatException e) { + logger.error("NumberFormatException when parse integer family number.str1:" + str1 + " str2:" + str2); + e.printStackTrace(); + return 0; + } + } else { //other numeric type + try { + Double num1 = Double.parseDouble(str1); + Double num2 = Double.parseDouble(str2); + return num1.compareTo(num2); + } catch (NumberFormatException e) { + logger.error("NumberFormatException when parse doul family number.str1:" + str1 + " str2:" + str2); + return 0; + } + } + } + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeByte(typeId); + text.write(dataOutput); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + this.typeId = dataInput.readByte(); + this.text = new Text(); + text.readFields(dataInput); + } + + public short getTypeId() { + return typeId; + } + + public Text getText() { + return text; + } + + public boolean isNumberFamily() { + if (typeId == TypeFlag.NONE_NUMERIC_TYPE.ordinal()) + return false; + return true; + } + + public boolean isIntegerFamily() { + return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal()); + } + + public boolean isOtherNumericFamily() { + return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal()); + } + + public void setTypeIdByDatatype(DataType type) { + if (!type.isNumberFamily()) { + this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal(); + } else if (type.isIntegerFamily()) { + this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal(); + } else { + this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal(); + } + } + + public void setTypeId(byte typeId) { + this.typeId = typeId; + } + + public void setText(Text text) { + this.text = text; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/e477b51f/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java index bb10b4a..bfaadb0 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java @@ -22,7 +22,7 @@ import org.apache.kylin.dict.NumberDictionaryBuilder; import org.apache.kylin.dict.NumberDictionaryForestBuilder; import org.apache.kylin.dict.StringBytesConverter; import org.apache.kylin.dict.TrieDictionaryForest; -import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag; +import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag; import org.junit.Ignore; import org.junit.Test; @@ -46,19 +46,19 @@ public class NumberDictionaryForestTest { private void testData(List<String> list, TypeFlag flag) { //stimulate map-reduce job - ArrayList<SelfDefineSortableKey> keyList = createKeyList(list, (byte) flag.ordinal()); + ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(list, (byte) flag.ordinal()); Collections.sort(keyList); //build tree NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(0, 0); - for (SelfDefineSortableKey key : keyList) { + for (SelfDefineSortableKeyOld key : keyList) { String fieldValue = printKey(key); b.addValue(fieldValue); } TrieDictionaryForest<String> dict = b.build(); dict.dump(System.out); ArrayList<Integer> resultIds = new ArrayList<>(); - for (SelfDefineSortableKey key : keyList) { + for (SelfDefineSortableKeyOld key : keyList) { String fieldValue = getFieldValue(key); resultIds.add(dict.getIdFromValue(fieldValue)); assertEquals(fieldValue, dict.getValueFromId(dict.getIdFromValue(fieldValue))); @@ -206,9 +206,9 @@ public class NumberDictionaryForestTest { return list; } - private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) { + private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList, byte typeFlag) { int partationId = 0; - ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>(); + ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>(); for (String str : strNumList) { ByteBuffer keyBuffer = ByteBuffer.allocate(4096); int offset = keyBuffer.position(); @@ -220,20 +220,20 @@ public class NumberDictionaryForestTest { //System.out.println("arrays toString:"+Arrays.toString(valueField)); Text outputKey = new Text(); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); - SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey); + SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag, outputKey); keyList.add(sortableKey); } return keyList; } - private String printKey(SelfDefineSortableKey key) { + private String printKey(SelfDefineSortableKeyOld key) { Text data = key.getText(); String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1); System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue); return fieldValue; } - private String getFieldValue(SelfDefineSortableKey key) { + private String getFieldValue(SelfDefineSortableKeyOld key) { Text data = key.getText(); return Bytes.toString(data.getBytes(), 1, data.getLength() - 1); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e477b51f/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java index d71e366..165fb83 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java @@ -12,7 +12,7 @@ import java.util.UUID; import org.apache.hadoop.io.Text; import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag; +import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag; import org.junit.Test; /** @@ -37,11 +37,11 @@ public class SelfDefineSortableKeyTest { System.out.println("test numbers:" + longList); ArrayList<String> strNumList = listToStringList(longList); //System.out.println("test num strs list:"+strNumList); - ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal()); + ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal()); System.out.println(keyList.get(0).isIntegerFamily()); Collections.sort(keyList); ArrayList<String> strListAftereSort = new ArrayList<>(); - for (SelfDefineSortableKey key : keyList) { + for (SelfDefineSortableKeyOld key : keyList) { String str = printKey(key); strListAftereSort.add(str); } @@ -73,11 +73,11 @@ public class SelfDefineSortableKeyTest { System.out.println("test numbers:" + doubleList); ArrayList<String> strNumList = listToStringList(doubleList); //System.out.println("test num strs list:"+strNumList); - ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal()); + ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal()); System.out.println(keyList.get(0).isOtherNumericFamily()); Collections.sort(keyList); ArrayList<String> strListAftereSort = new ArrayList<>(); - for (SelfDefineSortableKey key : keyList) { + for (SelfDefineSortableKeyOld key : keyList) { String str = printKey(key); strListAftereSort.add(str); } @@ -103,11 +103,11 @@ public class SelfDefineSortableKeyTest { strList.add("hello"); //duplicate strList.add("123"); strList.add(""); - ArrayList<SelfDefineSortableKey> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal()); + ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal()); System.out.println(keyList.get(0).isOtherNumericFamily()); Collections.sort(keyList); ArrayList<String> strListAftereSort = new ArrayList<>(); - for (SelfDefineSortableKey key : keyList) { + for (SelfDefineSortableKeyOld key : keyList) { String str = printKey(key); strListAftereSort.add(str); } @@ -138,10 +138,10 @@ public class SelfDefineSortableKeyTest { ArrayList<String> strNumList = listToStringList(doubleList); strNumList.add("fjaeif"); //illegal type //System.out.println("test num strs list:"+strNumList); - ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal()); + ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal()); System.out.println(keyList.get(0).isOtherNumericFamily()); Collections.sort(keyList); - for (SelfDefineSortableKey key : keyList) { + for (SelfDefineSortableKeyOld key : keyList) { printKey(key); } @@ -164,9 +164,9 @@ public class SelfDefineSortableKeyTest { return strList; } - private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) { + private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList, byte typeFlag) { int partationId = 0; - ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>(); + ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>(); for (String str : strNumList) { ByteBuffer keyBuffer = ByteBuffer.allocate(4096); int offset = keyBuffer.position(); @@ -175,13 +175,13 @@ public class SelfDefineSortableKeyTest { Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1); Text outputKey = new Text(); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); - SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey); + SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag, outputKey); keyList.add(sortableKey); } return keyList; } - private String printKey(SelfDefineSortableKey key) { + private String printKey(SelfDefineSortableKeyOld key) { Text data = key.getText(); String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1); System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);
