http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java deleted file mode 100644 index 03da261..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.Writable; - -public class BytesUtil { - - public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - public static void writeLong(long num, byte[] bytes, int offset, int size) { - for (int i = offset + size - 1; i >= offset; i--) { - bytes[i] = (byte) num; - num >>>= 8; - } - } - - public static void writeUnsigned(int num, byte[] bytes, int offset, int size) { - for (int i = offset + size - 1; i >= offset; i--) { - bytes[i] = (byte) num; - num >>>= 8; - } - } - - public static long readLong(byte[] bytes, int offset, int size) { - long integer = 0; - for (int i = offset, n = offset + size; i < n; i++) { - integer <<= 8; - integer |= (long) bytes[i] & 0xFF; - } - return integer; - } - - public static int readUnsigned(byte[] bytes, int offset, int size) { - int integer = 0; - for (int i = offset, n = offset + size; i < n; i++) { - integer <<= 8; - integer |= (int) bytes[i] & 0xFF; - } - return integer; - } - - public static void writeSigned(int num, byte[] bytes, int offset, int size) { - writeUnsigned(num, bytes, offset, size); - } - - public static int readSigned(byte[] bytes, int offset, int size) { - int integer = (bytes[offset] & 0x80) == 0 ? 0 : -1; - for (int i = offset, n = offset + size; i < n; i++) { - integer <<= 8; - integer |= (int) bytes[i] & 0xFF; - } - return integer; - } - - /** - * No. bytes needed to store a value as big as the given - */ - public static int sizeForValue(int maxValue) { - int size = 0; - while (maxValue > 0) { - size++; - maxValue >>>= 8; - } - return size; - } - - public static int compareByteUnsigned(byte b1, byte b2) { - int i1 = (int) b1 & 0xFF; - int i2 = (int) b2 & 0xFF; - return i1 - i2; - } - - public static byte[] subarray(byte[] bytes, int start, int end) { - byte[] r = new byte[end - start]; - System.arraycopy(bytes, start, r, 0, r.length); - return r; - } - - public static int compareBytes(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) { - int r = 0; - for (int i = 0; i < length; i++) { - r = src[srcOffset + i] - dst[dstOffset + i]; - if (r != 0) - break; - } - return r; - } - - // from WritableUtils - // ============================================================================ - - public static void writeVInt(int i, ByteBuffer out) { - writeVLong(i, out); - } - - public static void writeVLong(long i, ByteBuffer out) { - if (i >= -112 && i <= 127) { - out.put((byte) i); - return; - } - - int len = -112; - if (i < 0) { - i ^= -1L; // take one's complement' - len = -120; - } - - long tmp = i; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - - out.put((byte) len); - - len = (len < -120) ? -(len + 120) : -(len + 112); - - for (int idx = len; idx != 0; idx--) { - int shiftbits = (idx - 1) * 8; - long mask = 0xFFL << shiftbits; - out.put((byte) ((i & mask) >> shiftbits)); - } - } - - public static long readVLong(ByteBuffer in) { - byte firstByte = in.get(); - int len = decodeVIntSize(firstByte); - if (len == 1) { - return firstByte; - } - long i = 0; - for (int idx = 0; idx < len - 1; idx++) { - byte b = in.get(); - i = i << 8; - i = i | (b & 0xFF); - } - return (isNegativeVInt(firstByte) ? (i ^ -1L) : i); - } - - public static int readVInt(ByteBuffer in) { - long n = readVLong(in); - if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) { - throw new IllegalArgumentException("value too long to fit in integer"); - } - return (int) n; - } - - private static boolean isNegativeVInt(byte value) { - return value < -120 || (value >= -112 && value < 0); - } - - private static int decodeVIntSize(byte value) { - if (value >= -112) { - return 1; - } else if (value < -120) { - return -119 - value; - } - return -111 - value; - } - - public static void writeUnsigned(int num, int size, ByteBuffer out) { - for (int i = 0; i < size; i++) { - out.put((byte) num); - num >>>= 8; - } - } - - public static int readUnsigned(ByteBuffer in, int size) { - int integer = 0; - int mask = 0xff; - int shift = 0; - for (int i = 0; i < size; i++) { - integer |= (in.get() << shift) & mask; - mask = mask << 8; - shift += 8; - } - return integer; - } - - public static void writeLong(long num, ByteBuffer out) { - for (int i = 0; i < 8; i++) { - out.put((byte) num); - num >>>= 8; - } - } - - public static long readLong(ByteBuffer in) { - long integer = 0; - int mask = 0xff; - int shift = 0; - for (int i = 0; i < 8; i++) { - integer |= (in.get() << shift) & mask; - mask = mask << 8; - shift += 8; - } - return integer; - } - - public static void writeUTFString(String str, ByteBuffer out) { - byte[] bytes = str == null ? null : Bytes.toBytes(str); - writeByteArray(bytes, out); - } - - public static String readUTFString(ByteBuffer in) { - byte[] bytes = readByteArray(in); - return bytes == null ? null : Bytes.toString(bytes); - } - - - - public static void writeAsciiString(String str, ByteBuffer out) { - if (str == null) { - BytesUtil.writeVInt(-1, out); - return; - } - int len = str.length(); - BytesUtil.writeVInt(len, out); - for (int i = 0; i < len; i++) { - out.put((byte) str.charAt(i)); - } - } - - public static String readAsciiString(ByteBuffer in) { - int len = BytesUtil.readVInt(in); - if (len < 0) { - return null; - } - String result; - try { - if (in.hasArray()) { - int pos = in.position(); - result = new String(in.array(), pos, len, "ISO-8859-1"); - in.position(pos + len); - } else { - byte[] tmp = new byte[len]; - in.get(tmp); - result = new String(tmp, "ISO-8859-1"); - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - return result; - } - - public static void writeAsciiStringArray(String[] strs, ByteBuffer out) { - writeVInt(strs.length, out); - for (int i = 0; i < strs.length; i++) - writeAsciiString(strs[i], out); - } - - public static String[] readAsciiStringArray(ByteBuffer in) { - int len = readVInt(in); - String[] strs = new String[len]; - for (int i = 0; i < len; i++) - strs[i] = readAsciiString(in); - return strs; - } - - public static void writeIntArray(int[] array, ByteBuffer out) { - if (array == null) { - writeVInt(-1, out); - return; - } - writeVInt(array.length, out); - for (int i = 0; i < array.length; ++i) { - writeVInt(array[i], out); - } - } - - public static int[] readIntArray(ByteBuffer in) { - int len = readVInt(in); - if (len < 0) - return null; - int[] array = new int[len]; - - for (int i = 0; i < len; ++i) { - array[i] = readVInt(in); - } - return array; - } - - public static void writeByteArray(byte[] array, ByteBuffer out) { - if (array == null) { - writeVInt(-1, out); - return; - } - writeVInt(array.length, out); - out.put(array); - } - - public static byte[] readByteArray(ByteBuffer in) { - int len = readVInt(in); - if (len < 0) - return null; - - byte[] array = new byte[len]; - in.get(array); - return array; - } - - - public static int peekByteArrayLength(ByteBuffer in) { - int start = in.position(); - int arrayLen = readVInt(in); - int sizeLen = in.position() - start; - in.position(start); - - if (arrayLen < 0) - return sizeLen; - else - return sizeLen + arrayLen; - } - - - public static void writeBooleanArray(boolean[] array, ByteBuffer out) { - if (array == null) { - writeVInt(-1, out); - return; - } - writeVInt(array.length, out); - byte b_true = (byte) 1; - byte b_false = (byte) 0; - for (int i = 0; i < array.length; i++) { - if (array[i]) - out.put(b_true); - else - out.put(b_false); - } - } - - public static boolean[] readBooleanArray(ByteBuffer in) { - int len = readVInt(in); - if (len < 0) - return null; - - boolean[] array = new boolean[len]; - byte b_true = (byte) 1; - for (int i = 0; i < array.length; i++) { - byte temp = in.get(); - if (temp == b_true) - array[i] = true; - else - array[i] = false; - } - return array; - } - - public static byte[] toBytes(Writable writable) { - try { - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(bout); - writable.write(out); - out.close(); - bout.close(); - return bout.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public static String toReadableText(byte[] array) { - return toHex(array); - } - - /** - * this method only works for hex strings - */ - public static byte[] fromReadableText(String text) { - String[] tokens = text.split("\\\\x"); - byte[] ret = new byte[tokens.length - 1]; - for (int i = 1; i < tokens.length; ++i) { - int x = Bytes.toBinaryFromHex((byte) tokens[i].charAt(0)); - x = x << 4; - int y = Bytes.toBinaryFromHex((byte) tokens[i].charAt(1)); - ret[i - 1] = (byte) (x + y); - } - return ret; - } - - public static String toHex(byte[] array) { - return toHex(new ImmutableBytesWritable(array)); - } - - public static String toHex(ImmutableBytesWritable bytes) { - byte[] array = bytes.get(); - int offset = bytes.getOffset(); - int length = bytes.getLength(); - StringBuilder sb = new StringBuilder(length * 4); - for (int i = 0; i < length; i++) { - int b = array[offset + i]; - sb.append(String.format("\\x%02X", b & 0xFF)); - } - return sb.toString(); - } - - public static void main(String[] args) throws Exception { - } - -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java b/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java deleted file mode 100644 index 23e1973..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Created by qianzhou on 12/3/14. - */ -public class CaseInsensitiveStringMap<T> implements Map<String, T> { - - private Map<String, T> innerMap; - - public CaseInsensitiveStringMap() { - this(new HashMap<String, T>()); - } - - public CaseInsensitiveStringMap(Map<String, T> innerMap) { - this.innerMap = innerMap; - } - - @Override - public int size() { - return innerMap.size(); - } - - @Override - public boolean isEmpty() { - return innerMap.isEmpty(); - } - - @Override - public boolean containsKey(Object key) { - return key != null ? innerMap.containsKey(key.toString().toUpperCase()) : false; - } - - @Override - public boolean containsValue(Object value) { - return value != null ? innerMap.containsValue(value) : false; - } - - @Override - public T get(Object key) { - return key != null ? innerMap.get(key.toString().toUpperCase()) : null; - } - - @Override - public T put(String key, T value) { - return key != null ? innerMap.put(key.toString().toUpperCase(), value) : null; - } - - @Override - public T remove(Object key) { - return key != null ? innerMap.remove(key.toString().toUpperCase()) : null; - } - - @Override - public void putAll(Map<? extends String, ? extends T> m) { - innerMap.putAll(m); - } - - @Override - public void clear() { - innerMap.clear(); - } - - @Override - public Set<String> keySet() { - return innerMap.keySet(); - } - - @Override - public Collection<T> values() { - return innerMap.values(); - } - - @Override - public Set<Entry<String, T>> entrySet() { - return innerMap.entrySet(); - } - - @Override - public String toString() { - return this.innerMap.toString(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java deleted file mode 100644 index 7c8cda7..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.File; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.net.URL; -import java.net.URLClassLoader; - -/** - * @author xduo - * - */ -public class ClassUtil { - - public static void addClasspath(String path) throws Exception { - System.out.println("Adding path " + path + " to class path"); - File file = new File(path); - - if (file.exists()) { - URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader(); - Class<URLClassLoader> urlClass = URLClassLoader.class; - Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class }); - method.setAccessible(true); - method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() }); - } - } - - @SuppressWarnings("unchecked") - public static <T> Class<? extends T> forName(String name, Class<T> clz) throws ClassNotFoundException { - if (name.startsWith("com.kylinolap")) { - name = "org.apache.kylin" + name.substring("com.kylinolap".length()); - } - return (Class<? extends T>) Class.forName(name); - } - - public static Object newInstance(String clz) { - try { - return forName(clz, Object.class).newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static void updateFinalField(Class<?> clz, String fieldName, Object obj, Object newValue) { - try { - Field field = clz.getDeclaredField(fieldName); - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - field.set(obj, newValue); - - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java deleted file mode 100644 index 83fb6e0..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; - -import org.apache.commons.io.FileUtils; - -/** - * @author yangli9 - */ -public class CliCommandExecutor { - - private String remoteHost; - private String remoteUser; - private String remotePwd; - private int remoteTimeoutSeconds = 3600; - - public CliCommandExecutor() { - } - - public void setRunAtRemote(String host, String user, String pwd) { - this.remoteHost = host; - this.remoteUser = user; - this.remotePwd = pwd; - } - - public void setRunAtLocal() { - this.remoteHost = null; - this.remoteUser = null; - this.remotePwd = null; - } - - public void copyFile(String localFile, String destDir) throws IOException { - if (remoteHost == null) - copyNative(localFile, destDir); - else - copyRemote(localFile, destDir); - } - - private void copyNative(String localFile, String destDir) throws IOException { - File src = new File(localFile); - File dest = new File(destDir, src.getName()); - FileUtils.copyFile(src, dest); - } - - private void copyRemote(String localFile, String destDir) throws IOException { - SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd); - try { - ssh.scpFileToRemote(localFile, destDir); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException(e.getMessage(), e); - } - } - - public Pair<Integer, String> execute(String command) throws IOException { - return execute(command, new SoutLogger()); - } - - public Pair<Integer, String> execute(String command, Logger logAppender) throws IOException { - Pair<Integer, String> r; - if (remoteHost == null) { - r = runNativeCommand(command, logAppender); - } else { - r = runRemoteCommand(command, logAppender); - } - - if (r.getFirst() != 0) - throw new IOException("OS command error exit with " + r.getFirst() // - + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") // - + " -- " + command + "\n" + r.getSecond()); - - return r; - } - - private Pair<Integer, String> runRemoteCommand(String command, Logger logAppender) throws IOException { - SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd); - - SSHClientOutput sshOutput; - try { - sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender); - int exitCode = sshOutput.getExitCode(); - String output = sshOutput.getText(); - return new Pair<Integer, String>(exitCode, output); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException(e.getMessage(), e); - } - } - - private Pair<Integer, String> runNativeCommand(String command, Logger logAppender) throws IOException { - String[] cmd = new String[3]; - String osName = System.getProperty("os.name"); - if (osName.startsWith("Windows")) { - cmd[0] = "cmd.exe"; - cmd[1] = "/C"; - } else { - cmd[0] = "/bin/bash"; - cmd[1] = "-c"; - } - cmd[2] = command; - - ProcessBuilder builder = new ProcessBuilder(cmd); - builder.redirectErrorStream(true); - Process proc = builder.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); - String line; - StringBuilder result = new StringBuilder(); - while ((line = reader.readLine()) != null) { - result.append(line).append('\n'); - if (logAppender != null) { - logAppender.log(line); - } - } - - try { - int exitCode = proc.waitFor(); - return new Pair<Integer, String>(exitCode, result.toString()); - } catch (InterruptedException e) { - throw new IOException(e); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/DateFormat.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java deleted file mode 100644 index 69fc1a2..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.common.util; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; - -public class DateFormat { - - public static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd"; - public static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss"; - public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS"; - - static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>(); - - public static SimpleDateFormat getDateFormat(String datePattern) { - ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern); - if (formatThreadLocal == null) { - threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>()); - } - SimpleDateFormat format = formatThreadLocal.get(); - if (format == null) { - format = new SimpleDateFormat(datePattern); - format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly - formatThreadLocal.set(format); - } - return format; - } - - public static String formatToDateStr(long millis) { - return formatToDateStr(millis, DEFAULT_DATE_PATTERN); - } - - public static String formatToDateStr(long millis, String pattern) { - return getDateFormat(pattern).format(new Date(millis)); - } - - public static String formatToTimeStr(long millis) { - return getDateFormat(DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).format(new Date(millis)); - } - - public static String dateToString(Date date) { - return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS); - } - - public static String dateToString(Date date, String pattern) { - return getDateFormat(pattern).format(date); - } - - public static Date stringToDate(String str) { - return stringToDate(str, DEFAULT_DATE_PATTERN); - } - - public static Date stringToDate(String str, String pattern) { - Date date = null; - try { - date = getDateFormat(pattern).parse(str); - } catch (ParseException e) { - throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e); - } - return date; - } - - public static long stringToMillis(String str) { - if (isAllDigits(str)) { - return Long.parseLong(str); - } else if (str.length() == 10) { - return stringToDate(str, DEFAULT_DATE_PATTERN).getTime(); - } else if (str.length() == 19) { - return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime(); - } else if (str.length() == 23) { - return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime(); - } else { - throw new IllegalArgumentException("there is no valid date pattern for:" + str); - } - } - - private static boolean isAllDigits(String str) { - for (int i = 0, n = str.length(); i < n; i++) { - if (Character.isDigit(str.charAt(i)) == false) - return false; - } - return true; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Dictionary.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/common/src/main/java/org/apache/kylin/common/util/Dictionary.java deleted file mode 100644 index 0168609..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/Dictionary.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; - -import org.apache.hadoop.io.Writable; -import org.apache.kylin.common.util.BytesUtil; - -/** - * A bi-way dictionary that maps from dimension/column values to IDs and vice - * versa. By storing IDs instead of real values, the size of cube is - * significantly reduced. - * - * - IDs are smallest integers possible for the cardinality of a column, for the - * purpose of minimal storage space - IDs preserve ordering of values, such that - * range query can be applied to IDs directly - * - * A dictionary once built, is immutable. This allows optimal memory footprint - * by e.g. flatten the Trie structure into a byte array, replacing node pointers - * with array offsets. - * - * @author yangli9 - */ -abstract public class Dictionary<T> implements Writable { - - public static final byte NULL = (byte) 0xff; - - // ID with all bit-1 (0xff e.g.) reserved for NULL value - public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffff }; - - abstract public int getMinId(); - - abstract public int getMaxId(); - - public int getSize() { - return getMaxId() - getMinId() + 1; - } - - /** - * @return the size of an ID in bytes, determined by the cardinality of - * column - */ - abstract public int getSizeOfId(); - - /** - * @return the (maximum) size of value in bytes, determined by the longest - * value of column - */ - abstract public int getSizeOfValue(); - - /** - * Convenient form of <code>getIdFromValue(value, 0)</code> - */ - final public int getIdFromValue(T value) throws IllegalArgumentException { - return getIdFromValue(value, 0); - } - - /** - * Returns the ID integer of given value. In case of not found - * <p> - * - if roundingFlag=0, throw IllegalArgumentException; <br> - * - if roundingFlag<0, the closest smaller ID integer if exist; <br> - * - if roundingFlag>0, the closest bigger ID integer if exist. <br> - * <p> - * The implementation often has cache, thus faster than the byte[] version getIdFromValueBytes() - * - * @throws IllegalArgumentException - * if value is not found in dictionary and rounding is off or - * failed - */ - final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException { - if (isNullObjectForm(value)) - return nullId(); - else - return getIdFromValueImpl(value, roundingFlag); - } - - protected boolean isNullObjectForm(T value) { - return value == null; - } - - abstract protected int getIdFromValueImpl(T value, int roundingFlag); - - /** - * @return the value corresponds to the given ID - * @throws IllegalArgumentException - * if ID is not found in dictionary - */ - final public T getValueFromId(int id) { - if (isNullId(id)) - return null; - else - return getValueFromIdImpl(id); - } - - abstract protected T getValueFromIdImpl(int id); - - /** - * Convenient form of - * <code>getIdFromValueBytes(value, offset, len, 0)</code> - */ - final public int getIdFromValueBytes(byte[] value, int offset, int len) { - return getIdFromValueBytes(value, offset, len, 0); - } - - /** - * A lower level API, return ID integer from raw value bytes. In case of not found - * <p> - * - if roundingFlag=0, throw IllegalArgumentException; <br> - * - if roundingFlag<0, the closest smaller ID integer if exist; <br> - * - if roundingFlag>0, the closest bigger ID integer if exist. <br> - * <p> - * Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value). - * - * @throws IllegalArgumentException - * if value is not found in dictionary and rounding is off or failed - */ - final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) { - if (isNullByteForm(value, offset, len)) - return nullId(); - else - return getIdFromValueBytesImpl(value, offset, len, roundingFlag); - } - - protected boolean isNullByteForm(byte[] value, int offset, int len) { - return value == null; - } - - abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag); - - /** - * A lower level API, get byte values from ID, return the number of bytes - * written. Bypassing the cache layer, this could be significantly slower - * than getIdFromValue(T value). - * - * @return size of value bytes, 0 if empty string, -1 if null - * - * @throws IllegalArgumentException - * if ID is not found in dictionary - */ - final public int getValueBytesFromId(int id, byte[] returnValue, int offset) { - if (isNullId(id)) - return -1; - else - return getValueBytesFromIdImpl(id, returnValue, offset); - } - - abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset); - - abstract public void dump(PrintStream out); - - public int nullId() { - return NULL_ID[getSizeOfId()]; - } - - public boolean isNullId(int id) { - int nullId = NULL_ID[getSizeOfId()]; - return (nullId & id) == nullId; - } - - /** utility that converts a dictionary ID to string, preserving order */ - public static String dictIdToString(byte[] idBytes, int offset, int length) { - try { - return new String(idBytes, offset, length, "ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - // never happen - return null; - } - } - - /** the reverse of dictIdToString(), returns integer ID */ - public static int stringToDictId(String str) { - try { - byte[] bytes = str.getBytes("ISO-8859-1"); - return BytesUtil.readUnsigned(bytes, 0, bytes.length); - } catch (UnsupportedEncodingException e) { - // never happen - return 0; - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java b/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java deleted file mode 100644 index 093ac9e..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** This class will come with HBase 2.0 in package org.apache.hadoop.hbase.util **/ -package org.apache.kylin.common.util; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ClusterStatus; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.RegionLoad; -import org.apache.hadoop.hbase.ServerLoad; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class HBaseRegionSizeCalculator { - - private static final Logger logger = LoggerFactory.getLogger(HBaseRegionSizeCalculator.class); - - /** - * Maps each region to its size in bytes. - **/ - private final Map<byte[], Long> sizeMap = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR); - - static final String ENABLE_REGIONSIZECALCULATOR = "hbase.regionsizecalculator.enable"; - - /** - * Computes size of each region for table and given column families. - * */ - public HBaseRegionSizeCalculator(HTable table) throws IOException { - this(table, new HBaseAdmin(table.getConfiguration())); - } - - /** Constructor for unit testing */ - HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException { - - try { - if (!enabled(table.getConfiguration())) { - logger.info("Region size calculation disabled."); - return; - } - - logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\"."); - - // Get regions for table. - Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet(); - Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); - - for (HRegionInfo regionInfo : tableRegionInfos) { - tableRegions.add(regionInfo.getRegionName()); - } - - ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus(); - Collection<ServerName> servers = clusterStatus.getServers(); - final long megaByte = 1024L * 1024L; - - // Iterate all cluster regions, filter regions from our table and - // compute their size. - for (ServerName serverName : servers) { - ServerLoad serverLoad = clusterStatus.getLoad(serverName); - - for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) { - byte[] regionId = regionLoad.getName(); - - if (tableRegions.contains(regionId)) { - - long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte; - sizeMap.put(regionId, regionSizeBytes); - - // logger.info("Region " + regionLoad.getNameAsString() - // + " has size " + regionSizeBytes); - } - } - } - } finally { - hBaseAdmin.close(); - } - - } - - boolean enabled(Configuration configuration) { - return configuration.getBoolean(ENABLE_REGIONSIZECALCULATOR, true); - } - - /** - * Returns size of given region in bytes. Returns 0 if region was not found. - **/ - public long getRegionSize(byte[] regionId) { - Long size = sizeMap.get(regionId); - if (size == null) { - logger.info("Unknown region:" + Arrays.toString(regionId)); - return 0; - } else { - return size; - } - } - - public Map<byte[], Long> getRegionSizeMap() { - return Collections.unmodifiableMap(sizeMap); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java deleted file mode 100644 index 483a7a1..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.kylin.common.KylinConfig; - -public class HadoopUtil { - - private static ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>(); - - private static ThreadLocal<Configuration> hbaseConfig = new ThreadLocal<>(); - - public static void setCurrentConfiguration(Configuration conf) { - hadoopConfig.set(conf); - } - - public static void setCurrentHBaseConfiguration(Configuration conf) { - hbaseConfig.set(conf); - } - - public static Configuration getCurrentConfiguration() { - if (hadoopConfig.get() == null) { - hadoopConfig.set(newConfiguration()); - } - return hadoopConfig.get(); - } - - public static Configuration getCurrentHBaseConfiguration() { - if (hbaseConfig.get() == null) { - String storageUrl = KylinConfig.getInstanceFromEnv().getStorageUrl(); - hbaseConfig.set(newHBaseConfiguration(storageUrl)); - } - return hbaseConfig.get(); - } - - public static FileSystem getFileSystem(String path) throws IOException { - return FileSystem.get(makeURI(path), getCurrentConfiguration()); - } - - public static URI makeURI(String filePath) { - try { - return new URI(fixWindowsPath(filePath)); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e); - } - } - - public static String fixWindowsPath(String path) { - // fix windows path - if (path.startsWith("file://") && !path.startsWith("file:///") && path.contains(":\\")) { - path = path.replace("file://", "file:///"); - } - if (path.startsWith("file:///")) { - path = path.replace('\\', '/'); - } - return path; - } - - public static String makeQualifiedPathInHadoopCluster(String path) { - try { - FileSystem fs = FileSystem.get(getCurrentConfiguration()); - return fs.makeQualified(new Path(path)).toString(); - } catch (IOException e) { - throw new IllegalArgumentException("Cannot create FileSystem from current hadoop cluster conf", e); - } - } - - public static String makeQualifiedPathInHBaseCluster(String path) { - try { - FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration()); - return fs.makeQualified(new Path(path)).toString(); - } catch (IOException e) { - throw new IllegalArgumentException("Cannot create FileSystem from current hbase cluster conf", e); - } - } - - public static Configuration newConfiguration() { - Configuration conf = new Configuration(); - return healSickConfig(conf); - } - - public static Configuration newHBaseConfiguration(String url) { - Configuration conf = HBaseConfiguration.create(getCurrentConfiguration()); - - // using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath - if (!(StringUtils.isEmpty(url) || "hbase".equals(url))) - throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties"); - - // support hbase using a different FS - String hbaseClusterFs = KylinConfig.getInstanceFromEnv().getHBaseClusterFs(); - if (StringUtils.isNotEmpty(hbaseClusterFs)) { - conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hbaseClusterFs); - } - - // reduce rpc retry - conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); - conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5"); - conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); - // conf.set(ScannerCallable.LOG_SCANNER_ACTIVITY, "true"); - - return healSickConfig(conf); - } - - private static Configuration healSickConfig(Configuration conf) { - // https://issues.apache.org/jira/browse/KYLIN-953 - if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) { - conf.set("hadoop.tmp.dir", "/tmp"); - } - if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) { - conf.set("hbase.fs.tmp.dir", "/tmp"); - } - return conf; - } - - /** - * - * @param table the identifier of hive table, in format <db_name>.<table_name> - * @return a string array with 2 elements: {"db_name", "table_name"} - */ - public static String[] parseHiveTableName(String table) { - int cut = table.indexOf('.'); - String database = cut >= 0 ? table.substring(0, cut).trim() : "DEFAULT"; - String tableName = cut >= 0 ? table.substring(cut + 1).trim() : table.trim(); - - return new String[] { database, tableName }; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/HiveClient.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java deleted file mode 100644 index c56cb6c..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; -import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; - -/** - * Hive meta API client for Kylin - * @author shaoshi - * - */ -public class HiveClient { - - protected HiveConf hiveConf = null; - protected Driver driver = null; - protected HiveMetaStoreClient metaStoreClient = null; - protected String type; - - public HiveClient() { - hiveConf = new HiveConf(HiveClient.class); - } - - public HiveClient(Map<String, String> configMap) { - this(); - appendConfiguration(configMap); - } - - public HiveConf getHiveConf() { - return hiveConf; - } - - /** - * Get the hive ql driver to execute ddl or dml - * @return - */ - private Driver getDriver() { - if (driver == null) { - driver = new Driver(hiveConf); - SessionState.start(new CliSessionState(hiveConf)); - } - - return driver; - } - - /** - * Append or overwrite the default hive client configuration; You need call this before invoke #executeHQL; - * @param configMap - */ - public void appendConfiguration(Map<String, String> configMap) { - if (configMap != null && configMap.size() > 0) { - for (Entry<String, String> e : configMap.entrySet()) { - hiveConf.set(e.getKey(), e.getValue()); - } - } - } - - /** - * - * @param hql - * @throws org.apache.hadoop.hive.ql.CommandNeedRetryException - * @throws java.io.IOException - */ - public void executeHQL(String hql) throws CommandNeedRetryException, IOException { - CommandProcessorResponse response = getDriver().run(hql); - int retCode = response.getResponseCode(); - if (retCode != 0) { - String err = response.getErrorMessage(); - throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err); - } - } - - public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException { - for (String sql : hqls) - executeHQL(sql); - } - - private HiveMetaStoreClient getMetaStoreClient() throws Exception { - if (metaStoreClient == null) { - metaStoreClient = new HiveMetaStoreClient(hiveConf); - } - return metaStoreClient; - } - - public Table getHiveTable(String database, String tableName) throws Exception { - return getMetaStoreClient().getTable(database, tableName); - } - - public List<FieldSchema> getHiveTableFields(String database, String tableName) throws Exception { - return getMetaStoreClient().getFields(database, tableName); - } - - public String getHiveTableLocation(String database, String tableName) throws Exception { - Table t = getHiveTable(database, tableName); - return t.getSd().getLocation(); - } - - public long getFileSizeForTable(Table table) { - return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE); - } - - public long getFileNumberForTable(Table table) { - return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES); - } - - public List<String> getHiveDbNames() throws Exception { - return getMetaStoreClient().getAllDatabases(); - } - - public List<String> getHiveTableNames(String database) throws Exception { - return getMetaStoreClient().getAllTables(database); - } - - /** - * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility - * - * Get basic stats of table - * @param table - * - table - * @param statType - * - type of stats - * @return value of stats - */ - public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) { - Map<String, String> params = table.getParameters(); - long result = 0; - - if (params != null) { - try { - result = Long.parseLong(params.get(statType)); - } catch (NumberFormatException e) { - result = 0; - } - } - return result; - } - - public boolean isNativeTable(String database, String tableName) throws Exception { - return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName)); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java deleted file mode 100644 index 9b76899..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Reader; - -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; - -public class JsonUtil { - - // reuse the object mapper to save memory footprint - private static final ObjectMapper mapper = new ObjectMapper(); - private static final ObjectMapper indentMapper = new ObjectMapper(); - - static { - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true); - } - - public static <T> T readValue(File src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException { - return mapper.readValue(src, valueType); - } - - public static <T> T readValue(String content, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException { - return mapper.readValue(content, valueType); - } - - public static <T> T readValue(Reader src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException { - return mapper.readValue(src, valueType); - } - - public static <T> T readValue(InputStream src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException { - return mapper.readValue(src, valueType); - } - - public static <T> T readValue(byte[] src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException { - return mapper.readValue(src, valueType); - } - - public static void writeValueIndent(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException { - indentMapper.writeValue(out, value); - } - - public static void writeValue(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException { - mapper.writeValue(out, value); - } - - public static String writeValueAsString(Object value) throws JsonProcessingException { - return mapper.writeValueAsString(value); - } - - public static byte[] writeValueAsBytes(Object value) throws JsonProcessingException { - return mapper.writeValueAsBytes(value); - } - - public static String writeValueAsIndentString(Object value) throws JsonProcessingException { - return indentMapper.writeValueAsString(value); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java b/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java deleted file mode 100644 index 8653a6c..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.Enumeration; - -import org.apache.log4j.ConsoleAppender; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; - -/** - * Created by dongli on 11/24/15. - */ -public class Log4jConfigurer { - private final static String DEFAULT_PATTERN_LAYOUT = "L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n"; - private static boolean INITIALIZED = false; - - public static void initLogger() { - if (!INITIALIZED && !isConfigured()) { - org.apache.log4j.BasicConfigurator.configure(new ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT))); - } - INITIALIZED = true; - } - - private static boolean isConfigured() { - if (LogManager.getRootLogger().getAllAppenders().hasMoreElements()) { - return true; - } else { - Enumeration<?> loggers = LogManager.getCurrentLoggers(); - while (loggers.hasMoreElements()) { - Logger logger = (Logger) loggers.nextElement(); - if (logger.getAllAppenders().hasMoreElements()) - return true; - } - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java b/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java deleted file mode 100644 index e8a8dfb..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -/** - * Created by Hongbin Ma(Binmahone) on 1/27/15. - */ -public class LogTitlePrinter { - public static void printTitle(String title) { - String leftAlignFormat = "| %-100s | %n"; - - System.out.format("+------------------------------------------------------------------------------------------------------+%n"); - System.out.format(leftAlignFormat, title); - System.out.format("+------------------------------------------------------------------------------------------------------+%n"); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Logger.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/Logger.java b/common/src/main/java/org/apache/kylin/common/util/Logger.java deleted file mode 100644 index 0982b7b..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/Logger.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -/** - * @author ysong1 - * - */ -public interface Logger { - public void log(String message); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/MailService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/MailService.java b/common/src/main/java/org/apache/kylin/common/util/MailService.java deleted file mode 100644 index f3e385b..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/MailService.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.mail.Email; -import org.apache.commons.mail.EmailException; -import org.apache.commons.mail.HtmlEmail; -import org.apache.kylin.common.KylinConfig; - -/** - * @author xduo - */ -public class MailService { - - private Boolean enabled = Boolean.TRUE; - private String host; - private String username; - private String password; - private String sender; - - private static final Log logger = LogFactory.getLog(MailService.class); - - public MailService() { - this(KylinConfig.getInstanceFromEnv()); - } - - public MailService(KylinConfig config) { - enabled = "true".equalsIgnoreCase(config.getProperty(KylinConfig.MAIL_ENABLED, "false")); - host = config.getProperty(KylinConfig.MAIL_HOST, ""); - username = config.getProperty(KylinConfig.MAIL_USERNAME, ""); - password = config.getProperty(KylinConfig.MAIL_PASSWORD, ""); - sender = config.getProperty(KylinConfig.MAIL_SENDER, ""); - - if (enabled) { - if (host.isEmpty()) - throw new RuntimeException("mail service host is empty"); - } - } - - /** - * @param receivers - * @param subject - * @param content - * @return true or false indicating whether the email was delivered successfully - * @throws IOException - */ - public boolean sendMail(List<String> receivers, String subject, String content) throws IOException { - - if (!enabled) { - logger.info("Email service is disabled; this mail will not be delivered: " + subject); - logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties"); - return false; - } - - Email email = new HtmlEmail(); - email.setHostName(host); - if (username != null && username.trim().length() > 0) { - email.setAuthentication(username, password); - } - - //email.setDebug(true); - try { - for (String receiver : receivers) { - email.addTo(receiver); - } - - email.setFrom(sender); - email.setSubject(subject); - email.setCharset("UTF-8"); - ((HtmlEmail) email).setHtmlMsg(content); - email.send(); - email.getMailSession(); - - } catch (EmailException e) { - logger.error(e.getLocalizedMessage(), e); - return false; - } - - return true; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java b/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java deleted file mode 100644 index b4af6ce..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.sql.Timestamp; -import java.util.Date; -import java.util.logging.Formatter; -import java.util.logging.LogRecord; - -public class MyLogFormatter extends Formatter { - - Date dat = new Date(); - - // Line separator string. This is the value of the line.separator - // property at the moment that the SimpleFormatter was created. - private String lineSeparator = "\n"; - - /** - * Format the given LogRecord. - * - * @param record - * the log record to be formatted. - * @return a formatted log record - */ - public synchronized String format(LogRecord record) { - StringBuffer sb = new StringBuffer(); - // Minimize memory allocations here. - Timestamp ts = new Timestamp(record.getMillis()); - String text = ts.toString(); - sb.append("JUL "); - sb.append(text); - sb.append(" "); - if (record.getSourceClassName() != null) { - sb.append(record.getSourceClassName()); - } else { - sb.append(record.getLoggerName()); - } - if (record.getSourceMethodName() != null) { - sb.append(" "); - sb.append(record.getSourceMethodName()); - } - sb.append(lineSeparator); - String message = formatMessage(record); - sb.append(record.getLevel().getLocalizedName()); - sb.append(": "); - sb.append(message); - sb.append(lineSeparator); - if (record.getThrown() != null) { - try { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - record.getThrown().printStackTrace(pw); - pw.close(); - sb.append(sw.toString()); - } catch (Exception ex) { - } - } - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Pair.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/Pair.java b/common/src/main/java/org/apache/kylin/common/util/Pair.java deleted file mode 100644 index 17056af..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/Pair.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.Serializable; - -/** - * A generic class for pairs. Copied from org.apache.hadoop.hbase.util.Pair - * @param <T1> - * @param <T2> - */ -public class Pair<T1, T2> implements Serializable { - private static final long serialVersionUID = -3986244606585552569L; - protected T1 first = null; - protected T2 second = null; - - /** - * Default constructor. - */ - public Pair() { - } - - /** - * Constructor - * @param a operand - * @param b operand - */ - public Pair(T1 a, T2 b) { - this.first = a; - this.second = b; - } - - /** - * Constructs a new pair, inferring the type via the passed arguments - * @param <T1> type for first - * @param <T2> type for second - * @param a first element - * @param b second element - * @return a new pair containing the passed arguments - */ - public static <T1, T2> Pair<T1, T2> newPair(T1 a, T2 b) { - return new Pair<T1, T2>(a, b); - } - - /** - * Replace the first element of the pair. - * @param a operand - */ - public void setFirst(T1 a) { - this.first = a; - } - - /** - * Replace the second element of the pair. - * @param b operand - */ - public void setSecond(T2 b) { - this.second = b; - } - - /** - * Return the first element stored in the pair. - * @return T1 - */ - public T1 getFirst() { - return first; - } - - /** - * Return the second element stored in the pair. - * @return T2 - */ - public T2 getSecond() { - return second; - } - - private static boolean equals(Object x, Object y) { - return (x == null && y == null) || (x != null && x.equals(y)); - } - - @Override - @SuppressWarnings("unchecked") - public boolean equals(Object other) { - return other instanceof Pair && equals(first, ((Pair) other).first) && equals(second, ((Pair) other).second); - } - - @Override - public int hashCode() { - if (first == null) - return (second == null) ? 0 : second.hashCode() + 1; - else if (second == null) - return first.hashCode() + 2; - else - return first.hashCode() * 17 + second.hashCode(); - } - - @Override - public String toString() { - return "{" + getFirst() + "," + getSecond() + "}"; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java b/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java deleted file mode 100644 index a580302..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -import com.google.common.collect.Lists; - -/** - * Created by Hongbin Ma(Binmahone) on 1/5/15. - * - * This utility class sorts only the specified part of a list - */ -public class PartialSorter { - public static <T> void partialSort(List<T> list, List<Integer> items, Comparator<? super T> c) { - List<T> temp = Lists.newLinkedList(); - for (int index : items) { - temp.add(list.get(index)); - } - Collections.sort(temp, c); - for (int i = 0; i < temp.size(); ++i) { - list.set(items.get(i), temp.get(i)); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java b/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java deleted file mode 100644 index e0a31e4..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -/** - * @author ysong1 - * - */ -public class RandomSampler<T> { - - private Random rdm = new Random(); - - public List<T> sample(List<T> data, int sampleNumber) { - if (data == null) { - throw new IllegalArgumentException("Input list is null"); - } - if (data.size() < sampleNumber) { - return data; - } - - List<T> result = new ArrayList<T>(sampleNumber); - int n = data.size(); - for (int i = 0; i < n; i++) { - if (i < sampleNumber) { - result.add(data.get(i)); - } else { - int j = rdm.nextInt(i); - if (j < sampleNumber) { - result.set(j, data.get(i)); - } - } - } - return result; - } -}
