http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SSHClient.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/common/src/main/java/org/apache/kylin/common/util/SSHClient.java deleted file mode 100644 index 027045a..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -/** - * @author George Song (ysong1) - * - */ - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.slf4j.LoggerFactory; - -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelExec; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.Session; - -public class SSHClient { - protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(SSHClient.class); - - private String hostname; - private String username; - private String password; - private String identityPath; - - public SSHClient(String hostname, String username, String password) { - this.hostname = hostname; - this.username = username; - if (password != null && new File(password).exists()) { - this.identityPath = new File(password).getAbsolutePath(); - this.password = null; - } else { - this.password = password; - this.identityPath = null; - } - } - - public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception { - FileInputStream fis = null; - try { - System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory); - - Session session = newJSchSession(); - session.connect(); - - boolean ptimestamp = false; - - // exec 'scp -t rfile' remotely - String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory; - Channel channel = session.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - // get I/O streams for remote scp - OutputStream out = channel.getOutputStream(); - InputStream in = channel.getInputStream(); - - channel.connect(); - - if (checkAck(in) != 0) { - System.exit(0); - } - - File _lfile = new File(localFile); - - if (ptimestamp) { - command = "T " + (_lfile.lastModified() / 1000) + " 0"; - // The access time should be sent here, - // but it is not accessible with JavaAPI ;-< - command += (" " + (_lfile.lastModified() / 1000) + " 0\n"); - out.write(command.getBytes()); - out.flush(); - if (checkAck(in) != 0) { - throw new Exception("Error in checkAck()"); - } - } - - // send "C0644 filesize filename", where filename should not include '/' - long filesize = _lfile.length(); - command = "C0644 " + filesize + " "; - if (localFile.lastIndexOf("/") > 0) { - command += localFile.substring(localFile.lastIndexOf("/") + 1); - } else if (localFile.lastIndexOf(File.separator) > 0) { - command += localFile.substring(localFile.lastIndexOf(File.separator) + 1); - } else { - command += localFile; - } - command += "\n"; - out.write(command.getBytes()); - out.flush(); - if (checkAck(in) != 0) { - throw new Exception("Error in checkAck()"); - } - - // send a content of lfile - fis = new FileInputStream(localFile); - byte[] buf = new byte[1024]; - while (true) { - int len = fis.read(buf, 0, buf.length); - if (len <= 0) - break; - out.write(buf, 0, len); // out.flush(); - } - fis.close(); - fis = null; - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - if (checkAck(in) != 0) { - throw new Exception("Error in checkAck()"); - } - out.close(); - - channel.disconnect(); - session.disconnect(); - } catch (Exception e) { - throw e; - } finally { - try { - if (fis != null) - fis.close(); - } catch (Exception ee) { - } - } - } - - public void scpFileToLocal(String rfile, String lfile) throws Exception { - FileOutputStream fos = null; - try { - System.out.println("SCP remote file " + rfile + " to local " + lfile); - - String prefix = null; - if (new File(lfile).isDirectory()) { - prefix = lfile + File.separator; - } - - Session session = newJSchSession(); - session.connect(); - // exec 'scp -f rfile' remotely - String command = "scp -f " + rfile; - Channel channel = session.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - // get I/O streams for remote scp - OutputStream out = channel.getOutputStream(); - InputStream in = channel.getInputStream(); - - channel.connect(); - - byte[] buf = new byte[1024]; - - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - - while (true) { - int c = checkAck(in); - if (c != 'C') { - break; - } - - // read '0644 ' - in.read(buf, 0, 5); - - long filesize = 0L; - while (true) { - if (in.read(buf, 0, 1) < 0) { - // error - break; - } - if (buf[0] == ' ') - break; - filesize = filesize * 10L + (long) (buf[0] - '0'); - } - - String file = null; - for (int i = 0;; i++) { - in.read(buf, i, 1); - if (buf[i] == (byte) 0x0a) { - file = new String(buf, 0, i); - break; - } - } - - //System.out.println("filesize="+filesize+", file="+file); - - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - - // read a content of lfile - fos = new FileOutputStream(prefix == null ? lfile : prefix + file); - int foo; - while (true) { - if (buf.length < filesize) - foo = buf.length; - else - foo = (int) filesize; - foo = in.read(buf, 0, foo); - if (foo < 0) { - // error - break; - } - fos.write(buf, 0, foo); - filesize -= foo; - if (filesize == 0L) - break; - } - fos.close(); - fos = null; - - if (checkAck(in) != 0) { - System.exit(0); - } - - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - } - - session.disconnect(); - } catch (Exception e) { - throw e; - } finally { - try { - if (fos != null) - fos.close(); - } catch (Exception ee) { - } - } - } - - public SSHClientOutput execCommand(String command) throws Exception { - return execCommand(command, 7200, null); - } - - public SSHClientOutput execCommand(String command, int timeoutSeconds, Logger logAppender) throws Exception { - try { - System.out.println("[" + username + "@" + hostname + "] Execute command: " + command); - - StringBuffer text = new StringBuffer(); - int exitCode = -1; - - Session session = newJSchSession(); - session.connect(); - - Channel channel = session.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - channel.setInputStream(null); - - // channel.setOutputStream(System.out); - - ((ChannelExec) channel).setErrStream(System.err); - - InputStream in = channel.getInputStream(); - InputStream err = ((ChannelExec) channel).getErrStream(); - - channel.connect(); - - int timeout = timeoutSeconds; - byte[] tmp = new byte[1024]; - while (true) { - timeout--; - while (in.available() > 0) { - int i = in.read(tmp, 0, 1024); - if (i < 0) - break; - - String line = new String(tmp, 0, i); - text.append(line); - if (logAppender != null) { - logAppender.log(line); - } - } - while (err.available() > 0) { - int i = err.read(tmp, 0, 1024); - if (i < 0) - break; - - String line = new String(tmp, 0, i); - text.append(line); - if (logAppender != null) { - logAppender.log(line); - } - } - if (channel.isClosed()) { - if (in.available() > 0) - continue; - exitCode = channel.getExitStatus(); - System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode); - - break; - } - try { - Thread.sleep(1000); - } catch (Exception ee) { - throw ee; - } - if (timeout < 0) - throw new Exception("Remote commmand not finished within " + timeoutSeconds + " seconds."); - } - channel.disconnect(); - session.disconnect(); - return new SSHClientOutput(exitCode, text.toString()); - } catch (Exception e) { - throw e; - } - } - - private Session newJSchSession() throws JSchException { - JSch jsch = new JSch(); - if (identityPath != null) { - jsch.addIdentity(identityPath); - } - - Session session = jsch.getSession(username, hostname, 22); - if (password != null) { - session.setPassword(password); - } - session.setConfig("StrictHostKeyChecking", "no"); - return session; - } - - private int checkAck(InputStream in) throws IOException { - int b = in.read(); - // b may be 0 for success, - // 1 for error, - // 2 for fatal error, - // -1 - if (b == 0) - return b; - if (b == -1) - return b; - - if (b == 1 || b == 2) { - StringBuffer sb = new StringBuffer(); - int c; - do { - c = in.read(); - sb.append((char) c); - } while (c != '\n'); - if (b == 1) { // error - System.out.print(sb.toString()); - } - if (b == 2) { // fatal error - System.out.print(sb.toString()); - } - } - return b; - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java b/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java deleted file mode 100644 index a7e9e12..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -/** - * @author George Song (ysong1) - * - */ - -public class SSHClientOutput { - private String text; - private int exitCode = -1; - - /** - * @param text - * @param exitCode - */ - public SSHClientOutput(int exitCode, String text) { - this.text = text; - this.exitCode = exitCode; - } - - /** - * @return the text - */ - public String getText() { - return text.toString(); - } - - /** - * @return the exitCode - */ - public int getExitCode() { - return exitCode; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java b/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java deleted file mode 100644 index f82c8d6..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.common.util; - -/** - * Created by Hongbin Ma(Binmahone) on 2/6/15. - */ -public class SoutLogger implements Logger { - - @Override - public void log(String message) { - System.out.println(message); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java deleted file mode 100644 index 8deeb5f..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -/** - * @author George Song (ysong1) - * - */ -public class SplittedBytes { - public SplittedBytes(int length) { - this.value = new byte[length]; - this.length = 0; - } - - public byte[] value; - public int length; -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java b/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java deleted file mode 100644 index 4b7ba7e..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author George Song (ysong1) - * - */ -public class StringSplitter { - public static String[] split(String str, String delimiter) { - // The optimized split function - List<String> list = new ArrayList<String>(); - int index = 0, offset = 0; - int l = delimiter.length(); - if (str.startsWith(delimiter)) { - // in case the first field is empty - list.add(""); - offset = offset + l; - } - while ((index = str.indexOf(delimiter, index + 1)) != -1) { - list.add(str.substring(offset, index)); - offset = index + l; - } - // add the last field, or the str doesn't contain delimiter at all - list.add(str.substring(offset)); - return list.toArray(new String[0]); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/StringUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/common/src/main/java/org/apache/kylin/common/util/StringUtil.java deleted file mode 100644 index dbf3380..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.ArrayList; -import java.util.Collection; - -import org.apache.commons.lang.StringUtils; - -/** - * Created with IntelliJ IDEA. User: lukhan Date: 12/2/13 Time: 11:43 AM To - * change this template use File | Settings | File Templates. - */ -public class StringUtil { - - public static String[] filterSystemArgs(String args[]) { - ArrayList<String> whatsLeft = new ArrayList<String>(); - for (String a : args) { - if (a.startsWith("-D")) { - String key; - String value; - int cut = a.indexOf('='); - if (cut < 0) { - key = a.substring(2); - value = ""; - } else { - key = a.substring(2, cut); - value = a.substring(cut + 1); - } - System.setProperty(key, value); - } else { - whatsLeft.add(a); - } - } - return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]); - } - - public static void toUpperCaseArray(String[] source, String[] target) { - for (int i = 0; i < source.length; i++) { - if (source[i] != null) { - target[i] = source[i].toUpperCase(); - } - } - } - - public static String noBlank(String str, String dft) { - return StringUtils.isBlank(str) ? dft : str; - } - - public static String dropSuffix(String str, String suffix) { - if (str.endsWith(suffix)) - return str.substring(0, str.length() - suffix.length()); - else - return str; - } - - public static String min(Collection<String> strs) { - String min = null; - for (String s : strs) { - if (min == null || min.compareTo(s) > 0) - min = s; - } - return min; - } - - public static String max(Collection<String> strs) { - String max = null; - for (String s : strs) { - if (max == null || max.compareTo(s) < 0) - max = s; - } - return max; - } - - public static String min(String s1, String s2) { - if (s1 == null) - return s2; - else if (s2 == null) - return s1; - else - return s1.compareTo(s2) < 0 ? s1 : s2; - } - - public static String max(String s1, String s2) { - if (s1 == null) - return s2; - else if (s2 == null) - return s1; - else - return s1.compareTo(s2) > 0 ? s1 : s2; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java b/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java deleted file mode 100644 index 78751dc..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; - -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; - -public class TarGZUtil { - - public static void uncompressTarGZ(File tarFile, File dest) throws IOException { - dest.mkdir(); - TarArchiveInputStream tarIn = null; - - tarIn = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tarFile)))); - - TarArchiveEntry tarEntry = tarIn.getNextTarEntry(); - // tarIn is a TarArchiveInputStream - while (tarEntry != null) {// create a file with the same name as the tarEntry - File destPath = new File(dest, tarEntry.getName()); - System.out.println("working: " + destPath.getCanonicalPath()); - if (tarEntry.isDirectory()) { - destPath.mkdirs(); - } else { - destPath.createNewFile(); - //byte [] btoRead = new byte[(int)tarEntry.getSize()]; - byte[] btoRead = new byte[1024]; - //FileInputStream fin - // = new FileInputStream(destPath.getCanonicalPath()); - BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destPath)); - int len = 0; - - while ((len = tarIn.read(btoRead)) != -1) { - bout.write(btoRead, 0, len); - } - - bout.close(); - btoRead = null; - - } - tarEntry = tarIn.getNextTarEntry(); - } - tarIn.close(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java deleted file mode 100644 index 37c939c..0000000 --- a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.util.concurrent.Future; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * Created by Hongbin Ma(Binmahone) on 12/31/14. - */ -public class ThreadUtil { - @SuppressWarnings("unused") - public static void main(String[] args) { - ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable")); - - for (int i = 0; i < Integer.MAX_VALUE; ++i) { - System.out.println("index: " + i); - Future<?> future = pool.submit(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/resources/kylin-log4j.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/kylin-log4j.properties b/common/src/main/resources/kylin-log4j.properties deleted file mode 100644 index 3772972..0000000 --- a/common/src/main/resources/kylin-log4j.properties +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# enable this by -Dlog4j.configuration=kylin-log4j.properties - -log4j.rootLogger=INFO,stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n - -#log4j.logger.org.apache.hadoop=ERROR -log4j.logger.org.apache.kylin=DEBUG -log4j.logger.org.springframework=WARN http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java deleted file mode 100644 index 75625fb..0000000 --- a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.persistence; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.common.util.HadoopUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class HBaseResourceStoreTest extends HBaseMetadataTestCase { - - @Before - public void setup() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testHBaseStore() throws Exception { - testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv())); - } - - @Test - public void testHBaseStoreWithLargeCell() throws Exception { - String path = "/cube/_test_large_cell.json"; - String largeContent = "THIS_IS_A_LARGE_CELL"; - StringEntity content = new StringEntity(largeContent); - KylinConfig config = KylinConfig.getInstanceFromEnv(); - int origSize = config.getHBaseKeyValueSize(); - ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - - try { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1)); - - store.deleteResource(path); - - store.putResource(path, content, StringEntity.serializer); - assertTrue(store.exists(path)); - StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer); - assertEquals(content, t); - - Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path); - Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration(); - FileSystem fileSystem = FileSystem.get(hconf); - assertTrue(fileSystem.exists(redirectPath)); - - FSDataInputStream in = fileSystem.open(redirectPath); - assertEquals(largeContent, in.readUTF()); - in.close(); - - store.deleteResource(path); - } finally { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize); - store.deleteResource(path); - } - } - - void testAStore(ResourceStore store) throws IOException { - String dir1 = "/cube"; - String path1 = "/cube/_test.json"; - StringEntity content1 = new StringEntity("anything"); - String dir2 = "/table"; - String path2 = "/table/_test.json"; - StringEntity content2 = new StringEntity("something"); - - // cleanup legacy if any - store.deleteResource(path1); - store.deleteResource(path2); - - StringEntity t; - - // put/get - store.putResource(path1, content1, StringEntity.serializer); - assertTrue(store.exists(path1)); - t = store.getResource(path1, StringEntity.class, StringEntity.serializer); - assertEquals(content1, t); - - store.putResource(path2, content2, StringEntity.serializer); - assertTrue(store.exists(path2)); - t = store.getResource(path2, StringEntity.class, StringEntity.serializer); - assertEquals(content2, t); - - // overwrite - t.str = "new string"; - store.putResource(path2, t, StringEntity.serializer); - - // write conflict - try { - t.setLastModified(t.lastModified - 1); - store.putResource(path2, t, StringEntity.serializer); - fail("write conflict should trigger IllegalStateException"); - } catch (IllegalStateException e) { - // expected - } - - // list - ArrayList<String> list; - - list = store.listResources(dir1); - assertTrue(list.contains(path1)); - assertTrue(list.contains(path2) == false); - - list = store.listResources(dir2); - assertTrue(list.contains(path2)); - assertTrue(list.contains(path1) == false); - - list = store.listResources("/"); - assertTrue(list.contains(dir1)); - assertTrue(list.contains(dir2)); - assertTrue(list.contains(path1) == false); - assertTrue(list.contains(path2) == false); - - list = store.listResources(path1); - assertNull(list); - list = store.listResources(path2); - assertNull(list); - - // delete/exist - store.deleteResource(path1); - assertTrue(store.exists(path1) == false); - list = store.listResources(dir1); - assertTrue(list == null || list.contains(path1) == false); - - store.deleteResource(path2); - assertTrue(store.exists(path2) == false); - list = store.listResources(dir2); - assertTrue(list == null || list.contains(path2) == false); - } - - public static class StringEntity extends RootPersistentEntity { - - static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() { - @Override - public void serialize(StringEntity obj, DataOutputStream out) throws IOException { - out.writeUTF(obj.str); - } - - @Override - public StringEntity deserialize(DataInputStream in) throws IOException { - String str = in.readUTF(); - return new StringEntity(str); - } - }; - - String str; - - public StringEntity(String str) { - this.str = str; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((str == null) ? 0 : str.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (obj == this) - return true; - if (!(obj instanceof StringEntity)) - return false; - return StringUtils.equals(this.str, ((StringEntity) obj).str); - } - - @Override - public String toString() { - return str; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java deleted file mode 100644 index 4a9daa6..0000000 --- a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.persistence; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase { - - @Before - public void setup() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testFileStore() throws Exception { - testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv())); - } - - void testAStore(ResourceStore store) throws IOException { - String dir1 = "/cube"; - String path1 = "/cube/_test.json"; - StringEntity content1 = new StringEntity("anything"); - String dir2 = "/table"; - String path2 = "/table/_test.json"; - StringEntity content2 = new StringEntity("something"); - - // cleanup legacy if any - store.deleteResource(path1); - store.deleteResource(path2); - - StringEntity t; - - // put/get - store.putResource(path1, content1, StringEntity.serializer); - assertTrue(store.exists(path1)); - t = store.getResource(path1, StringEntity.class, StringEntity.serializer); - assertEquals(content1, t); - - store.putResource(path2, content2, StringEntity.serializer); - assertTrue(store.exists(path2)); - t = store.getResource(path2, StringEntity.class, StringEntity.serializer); - assertEquals(content2, t); - - // overwrite - t.str = "new string"; - store.putResource(path2, t, StringEntity.serializer); - - // write conflict - try { - t.setLastModified(t.lastModified - 1); - store.putResource(path2, t, StringEntity.serializer); - fail("write conflict should trigger IllegalStateException"); - } catch (IllegalStateException e) { - // expected - } - - // list - ArrayList<String> list; - - list = store.listResources(dir1); - assertTrue(list.contains(path1)); - assertTrue(list.contains(path2) == false); - - list = store.listResources(dir2); - assertTrue(list.contains(path2)); - assertTrue(list.contains(path1) == false); - - list = store.listResources("/"); - assertTrue(list.contains(dir1)); - assertTrue(list.contains(dir2)); - assertTrue(list.contains(path1) == false); - assertTrue(list.contains(path2) == false); - - list = store.listResources(path1); - assertNull(list); - list = store.listResources(path2); - assertNull(list); - - // delete/exist - store.deleteResource(path1); - assertTrue(store.exists(path1) == false); - list = store.listResources(dir1); - assertTrue(list == null || list.contains(path1) == false); - - store.deleteResource(path2); - assertTrue(store.exists(path2) == false); - list = store.listResources(dir2); - assertTrue(list == null || list.contains(path2) == false); - } - - public static class StringEntity extends RootPersistentEntity { - - static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() { - @Override - public void serialize(StringEntity obj, DataOutputStream out) throws IOException { - out.writeUTF(obj.str); - } - - @Override - public StringEntity deserialize(DataInputStream in) throws IOException { - String str = in.readUTF(); - return new StringEntity(str); - } - }; - - String str; - - public StringEntity(String str) { - this.str = str; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((str == null) ? 0 : str.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (obj == this) - return true; - if (!(obj instanceof StringEntity)) - return false; - return StringUtils.equals(this.str, ((StringEntity) obj).str); - } - - @Override - public String toString() { - return str; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java deleted file mode 100644 index 14538c6..0000000 --- a/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.persistence; - -import java.io.File; -import java.io.IOException; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.ClassUtil; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Created by honma on 9/18/14. - */ -@Ignore("convenient trial tool for dev") -public class ResourceToolTest { - @Before - public void setup() throws Exception { - ClassUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath()); - } - - @Test - public void test() throws IOException { - ResourceTool.copy(KylinConfig.createInstanceFromUri("../examples/test_case_data"), KylinConfig.createInstanceFromUri("../examples/test_case_data/kylin.properties")); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java b/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java deleted file mode 100644 index f4d3410..0000000 --- a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.restclient; - -import java.io.IOException; - -import org.junit.Test; - -public class RestClientTest { - - @SuppressWarnings("unused") - @Test - public void basicTests() throws IOException { - RestClient a = new RestClient("prod01:80"); - //a.wipeCache("metadata", "a", "a"); - //String aa = a.getKylinProperties(); - //System.out.println(aa); - RestClient b = new RestClient("sandbox.hortonworks.com:7070"); - //b.wipeCache("metadata", "a", "a"); - //String bb = b.getKylinProperties(); - //System.out.println(bb); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java deleted file mode 100644 index 58d3ac1..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.lang.reflect.Method; - -import org.apache.kylin.common.KylinConfig; - -/** - * @author ysong1 - * - */ -public abstract class AbstractKylinTestCase { - - public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta"; - - public static final String LOCALMETA_TEST_DATA_V1 = "../examples/test_case_data/localmeta_v1"; - - public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster"; - - public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox"; - - public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.metadata.project.ProjectManager" }; - - public abstract void createTestMetadata() throws Exception; - - public abstract void cleanupTestMetadata() throws Exception; - - public static KylinConfig getTestConfig() { - return KylinConfig.getInstanceFromEnv(); - } - - public static void staticCreateTestMetadata(String kylinConfigFolder) { - - KylinConfig.destoryInstance(); - - HadoopUtil.setCurrentConfiguration(null); - HadoopUtil.setCurrentHBaseConfiguration(null); - if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null) - System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder); - - } - - public static void staticCleanupTestMetadata() { - cleanupCache(); - System.clearProperty(KylinConfig.KYLIN_CONF); - KylinConfig.destoryInstance(); - - HadoopUtil.setCurrentConfiguration(null); - HadoopUtil.setCurrentHBaseConfiguration(null); - - } - - private static void cleanupCache() { - - for (String serviceClass : SERVICES_WITH_CACHE) { - try { - Class<?> cls = Class.forName(serviceClass); - Method method = cls.getDeclaredMethod("clearCache"); - method.invoke(null); - } catch (ClassNotFoundException e) { - // acceptable because lower module test does have CubeManager etc on classpath - } catch (Exception e) { - System.err.println("Error clean up cache " + serviceClass); - e.printStackTrace(); - } - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java deleted file mode 100644 index 6d2762c..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Created by honma on 11/11/14. - * - * development concept proving use - */ -@Ignore("convenient trial tool for dev") -public class BasicHadoopTest { - - @BeforeClass - public static void setup() throws Exception { - ClassUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath()); - } - - @Test - public void testCreateHtable() throws IOException { - HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf("testhbase")); - tableDesc.setValue("KYLIN_HOST", "dev01"); - - HColumnDescriptor cf = new HColumnDescriptor("f"); - cf.setMaxVersions(1); - - cf.setInMemory(true); - cf.setBlocksize(4 * 1024 * 1024); // set to 4MB - tableDesc.addFamily(cf); - - Configuration conf = HBaseConfiguration.create(); - HBaseAdmin admin = new HBaseAdmin(conf); - admin.createTable(tableDesc); - admin.close(); - } - - @Test - public void testRetriveHtableHost() throws IOException { - Configuration conf = HBaseConfiguration.create(); - HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); - HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(); - for (HTableDescriptor table : tableDescriptors) { - String value = table.getValue("KYLIN_HOST"); - if (value != null) { - System.out.println(table.getTableName()); - System.out.println("host is " + value); - hbaseAdmin.disableTable(table.getTableName()); - table.setValue("KYLIN_HOST_ANOTHER", "dev02"); - hbaseAdmin.modifyTable(table.getTableName(), table); - hbaseAdmin.enableTable(table.getTableName()); - } - } - hbaseAdmin.close(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java deleted file mode 100644 index f2778d2..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.commons.configuration.ConfigurationException; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.LoggerFactory; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; - -/** -* Created by honma on 10/17/14. -* <p/> -* Keep this test case to test basic java functionality -* development concept proving use -*/ -@Ignore("convenient trial tool for dev") -@SuppressWarnings("unused") -public class BasicTest { - protected static final org.slf4j.Logger log = LoggerFactory.getLogger(BasicTest.class); - - private void log(ByteBuffer a) { - Integer x = 4; - foo(x); - } - - private void foo(Long a) { - System.out.printf("a"); - - } - - private void foo(Integer b) { - System.out.printf("b"); - } - - private enum MetricType { - Count, DimensionAsMetric, DistinctCount, Normal - } - - @Test - @Ignore("convenient trial tool for dev") - public void test1() throws Exception { - ExecutorService executorService = Executors.newFixedThreadPool(10); - ListenableFuture futureTask = MoreExecutors.listeningDecorator(executorService).submit(new Runnable() { - @Override - public void run() { - - } - }); - futureTask.addListener(new Runnable() { - @Override - public void run() { - - } - }, executorService); - - } - - @Test - @Ignore("fix it later") - public void test2() throws IOException, ConfigurationException { - int m = 1 << 15; - System.out.println(m); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java deleted file mode 100644 index 178e568..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.nio.ByteBuffer; - -import junit.framework.TestCase; - -import org.junit.Test; - -/** - * by honma - */ -public class BytesUtilTest extends TestCase { - @Test - public void test() { - ByteBuffer buffer = ByteBuffer.allocate(10000); - int[] x = new int[] { 1, 2, 3 }; - BytesUtil.writeIntArray(x, buffer); - buffer.flip(); - - byte[] buf = new byte[buffer.limit()]; - System.arraycopy(buffer.array(), 0, buf, 0, buffer.limit()); - - ByteBuffer newBuffer = ByteBuffer.wrap(buf); - int[] y = BytesUtil.readIntArray(newBuffer); - assertEquals(y[2], 3); - } - - @Test - public void testBooleanArray() { - ByteBuffer buffer = ByteBuffer.allocate(10000); - boolean[] x = new boolean[] { true, false, true }; - BytesUtil.writeBooleanArray(x, buffer); - buffer.flip(); - boolean[] y = BytesUtil.readBooleanArray(buffer); - assertEquals(y[2], true); - assertEquals(y[1], false); - } - - @Test - public void testReadable() { - String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8"; - byte[] bytes = BytesUtil.fromReadableText(x); - String y = BytesUtil.toHex(bytes); - assertEquals(x, y); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java deleted file mode 100644 index d2e3238..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.IOException; - -/** - * @author ysong1 - */ -public class HBaseMetadataTestCase extends AbstractKylinTestCase { - - static { - if (useSandbox()) { - try { - File sandboxFolder = new File("../examples/test_case_data/sandbox/"); - if (sandboxFolder.exists() == false) { - throw new IOException("The sandbox folder doesn't exist: " + sandboxFolder.getAbsolutePath()); - } - ClassUtil.addClasspath(sandboxFolder.getAbsolutePath()); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - @Override - public void createTestMetadata() throws Exception { - staticCreateTestMetadata(); - } - - @Override - public void cleanupTestMetadata() { - staticCleanupTestMetadata(); - } - - public static void staticCreateTestMetadata() throws Exception { - if (useSandbox()) { - staticCreateTestMetadata(SANDBOX_TEST_DATA); - } else { - staticCreateTestMetadata(MINICLUSTER_TEST_DATA); - HBaseMiniclusterHelper.startupMinicluster(); - } - - } - - public static boolean useSandbox() { - String useSandbox = System.getProperty("useSandbox"); - if (StringUtils.isEmpty(useSandbox)) { - return true; - } - - return Boolean.parseBoolean(useSandbox); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java deleted file mode 100644 index 6f31776..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.HBaseResourceStore; - -/** - * a helper class to start and shutdown hbase mini cluster - * - * @author shaoshi - */ -public class HBaseMiniclusterHelper { - - public static final String SHARED_STORAGE_PREFIX = "KYLIN_"; - public static final String CUBE_STORAGE_PREFIX = "KYLIN_"; - public static final String II_STORAGE_PREFIX = "KYLIN_II"; - public static final String TEST_METADATA_TABLE = "kylin_metadata"; - - private static final String hbaseTarLocation = "../examples/test_case_data/minicluster/hbase-export.tar.gz"; - private static final String iiEndpointClassName = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint"; - - public static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static volatile boolean clusterStarted = false; - private static String zkHost, zkPort, zkParent; - - private static final Log logger = LogFactory.getLog(HBaseMiniclusterHelper.class); - - static { - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - shutdownMiniCluster(); - } - }); - } - - /** - * Start the minicluster; Sub-classes should invoke this in BeforeClass method. - * - * @throws Exception - */ - public static void startupMinicluster() throws Exception { - HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.MINICLUSTER_TEST_DATA); - - if (!clusterStarted) { - synchronized (HBaseMiniclusterHelper.class) { - if (!clusterStarted) { - startupMiniClusterAndImportData(); - clusterStarted = true; - } - } - } else { - updateKylinConfigWithMinicluster(); - } - } - - private static void updateKylinConfigWithMinicluster() { - - Configuration hbaseConfiguration = HadoopUtil.getCurrentHBaseConfiguration(); - hbaseConfiguration.set(HConstants.ZOOKEEPER_QUORUM, zkHost); - hbaseConfiguration.set(HConstants.ZOOKEEPER_CLIENT_PORT, zkPort); - hbaseConfiguration.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkParent); - } - - private static void startupMiniClusterAndImportData() throws Exception { - - logger.info("Going to start mini cluster."); - - if (existInClassPath(iiEndpointClassName)) { - HBaseMiniclusterHelper.UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, iiEndpointClassName); - } - - //https://issues.apache.org/jira/browse/HBASE-11711 - UTIL.getConfiguration().setInt("hbase.master.info.port", -1);//avoid port clobbering - - MiniHBaseCluster hbaseCluster = UTIL.startMiniCluster(); - UTIL.startMiniMapReduceCluster(); - Configuration config = hbaseCluster.getConf(); - zkHost = config.get(HConstants.ZOOKEEPER_QUORUM); - zkPort = config.get(HConstants.ZOOKEEPER_CLIENT_PORT); - zkParent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT); - - // see in: https://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired - config.set("zookeeper.session.timeout", "1200000"); - config.set("hbase.zookeeper.property.tickTime", "6000"); - // reduce rpc retry - config.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); - config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1"); - config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); - - updateKylinConfigWithMinicluster(); - KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - // create the metadata htables; - @SuppressWarnings("unused") - HBaseResourceStore store = new HBaseResourceStore(kylinConfig); - - // import the table content - HbaseImporter.importHBaseData(hbaseTarLocation, UTIL.getConfiguration()); - - } - - private static boolean existInClassPath(String className) { - try { - Class.forName(className); - } catch (ClassNotFoundException e) { - return false; - } - return true; - } - - /** - * Shutdown the minicluster; - */ - public static void shutdownMiniCluster() { - - logger.info("Going to shutdown mini cluster."); - - try { - UTIL.shutdownMiniMapReduceCluster(); - } catch (Exception e) { - e.printStackTrace(); - } - - try { - UTIL.shutdownMiniCluster(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public static void main(String[] args) { - HBaseMiniclusterHelper t = new HBaseMiniclusterHelper(); - logger.info(t); - try { - HBaseMiniclusterHelper.startupMinicluster(); - } catch (Exception e) { - e.printStackTrace(); - } finally { - HBaseMiniclusterHelper.shutdownMiniCluster(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java deleted file mode 100644 index 8587683..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.kylin.common.KylinConfig; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; - - -import static org.junit.Assert.*; - -/** - * Created by sunyerui on 15/8/26. - * Tests for HadoopUtil - */ -public class HadoopUtilTest { - - @BeforeClass - public static void beforeClass() { - System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - } - - @After - public void after() { - HadoopUtil.setCurrentConfiguration(null); - HadoopUtil.setCurrentHBaseConfiguration(null); - } - - @Test - public void testGetCurrentHBaseConfiguration() throws Exception { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "hdfs://hbase-cluster/"); - - Configuration conf = HadoopUtil.getCurrentHBaseConfiguration(); - assertEquals("hdfs://hbase-cluster/", conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); - } - - @Test - public void testMakeQualifiedPathInHBaseCluster() throws Exception { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "file:/"); - - String path = HadoopUtil.makeQualifiedPathInHBaseCluster("/path/to/test/hbase"); - assertEquals("file:/path/to/test/hbase", path); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java deleted file mode 100644 index 5d8d6d6..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.mapreduce.Import; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.util.GenericOptionsParser; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.HBaseConnection; - -import com.google.common.base.Preconditions; - -public class HbaseImporter { - - private static final Log logger = LogFactory.getLog(HbaseImporter.class); - - public static void importHBaseData(String hbaseTarLocation, Configuration conf) throws IOException, ClassNotFoundException, InterruptedException { - - if (System.getenv("JAVA_HOME") == null) { - logger.error("Didn't find $JAVA_HOME, this will cause HBase data import failed. Please set $JAVA_HOME."); - logger.error("Skipping table import..."); - return; - } - - File exportFile = new File(hbaseTarLocation); - if (!exportFile.exists()) { - logger.error("Didn't find the export achieve file on " + exportFile.getAbsolutePath()); - return; - } - - File folder = File.createTempFile("hbase-import", "tmp"); - if (folder.exists()) { - FileUtils.forceDelete(folder); - } - folder.mkdirs(); - FileUtils.forceDeleteOnExit(folder); - - //TarGZUtil.uncompressTarGZ(exportFile, folder); - FileUtil.unTar(exportFile, folder); - String[] child = folder.list(); - Preconditions.checkState(child.length == 1); - String backupFolderName = child[0]; - File backupFolder = new File(folder, backupFolderName); - String[] tableNames = backupFolder.list(); - - for (String table : tableNames) { - - if (!(table.equalsIgnoreCase(HBaseMiniclusterHelper.TEST_METADATA_TABLE) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) { - continue; - } - - // create the htable; otherwise the import will fail. - if (table.startsWith(HBaseMiniclusterHelper.II_STORAGE_PREFIX)) { - HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "f"); - } else if (table.startsWith(HBaseMiniclusterHelper.CUBE_STORAGE_PREFIX)) { - HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2"); - } - - // directly import from local fs, no need to copy to hdfs - String importLocation = "file://" + backupFolder.getAbsolutePath() + "/" + table; - String[] args = new String[] { table, importLocation }; - boolean result = runImport(args, conf); - logger.info("importing table '" + table + "' with result:" + result); - - if (!result) - break; - } - - } - - private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException { - // need to make a copy of the configuration because to make sure different temp dirs are used. - GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args); - Configuration newConf = opts.getConfiguration(); - args = opts.getRemainingArgs(); - Job job = Import.createSubmittableJob(newConf, args); - job.waitForCompletion(false); - return job.isSuccessful(); - } - - public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { - if (args.length != 1) { - logger.error("Usage: HbaseImporter hbase_tar_lcoation"); - System.exit(-1); - } - - logger.info("The KylinConfig being used:"); - logger.info("================================================="); - KylinConfig.getInstanceFromEnv().printProperties(); - logger.info("================================================="); - - importHBaseData(args[0], HBaseConfiguration.create()); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java deleted file mode 100644 index ff42713..0000000 --- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashSet; -import java.util.Random; -import java.util.Set; - -import org.apache.kylin.common.hll.HyperLogLogPlusCounter; -import org.junit.Assert; -import org.junit.Test; - -/** - * @author yangli9 - * - */ -public class HyperLogLogCounterTest { - - ByteBuffer buf = ByteBuffer.allocate(1024 * 1024); - Random rand1 = new Random(1); - Random rand2 = new Random(2); - Random rand3 = new Random(3); - int errorCount1 = 0; - int errorCount2 = 0; - int errorCount3 = 0; - - private Set<String> generateTestData(int n) { - Set<String> testData = new HashSet<String>(); - for (int i = 0; i < n; i++) { - String[] samples = generateSampleData(); - for (String sample : samples) { - testData.add(sample); - } - } - return testData; - } - - // simulate the visit (=visitor+id) - private String[] generateSampleData() { - - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < 19; i++) { - buf.append(Math.abs(rand1.nextInt()) % 10); - } - String header = buf.toString(); - - int size = Math.abs(rand3.nextInt()) % 9 + 1; - String[] samples = new String[size]; - for (int k = 0; k < size; k++) { - buf = new StringBuilder(header); - buf.append("-"); - for (int i = 0; i < 10; i++) { - buf.append(Math.abs(rand3.nextInt()) % 10); - } - samples[k] = buf.toString(); - } - - return samples; - } - - @Test - public void countTest() throws IOException { - int n = 10; - for (int i = 0; i < 5; i++) { - count(n); - n *= 10; - } - } - - private void count(int n) throws IOException { - Set<String> testSet = generateTestData(n); - - HyperLogLogPlusCounter hllc = newHLLC(); - for (String testData : testSet) { - hllc.add(Bytes.toBytes(testData)); - } - long estimate = hllc.getCountEstimate(); - double errorRate = hllc.getErrorRate(); - double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size(); - System.out.println(estimate); - System.out.println(testSet.size()); - System.out.println(errorRate); - System.out.println("=" + actualError); - Assert.assertTrue(actualError < errorRate * 3.0); - - checkSerialize(hllc); - } - - private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException { - long estimate = hllc.getCountEstimate(); - buf.clear(); - hllc.writeRegisters(buf); - buf.flip(); - hllc.readRegisters(buf); - Assert.assertEquals(estimate, hllc.getCountEstimate()); - } - - @Test - public void mergeTest() throws IOException { - double error = 0; - double absError = 0; - int n = 100; - for (int i = 0; i < n; i++) { - double e = merge(); - error += e; - absError += Math.abs(e); - } - System.out.println("Total average error is " + error / n + " and absolute error is " + absError / n); - - System.out.println(" errorRateCount1 is " + errorCount1 + "!"); - System.out.println(" errorRateCount2 is " + errorCount2 + "!"); - System.out.println(" errorRateCount3 is " + errorCount3 + "!"); - - Assert.assertTrue(errorCount1 <= n * 0.40); - Assert.assertTrue(errorCount2 <= n * 0.08); - Assert.assertTrue(errorCount3 <= n * 0.02); - } - - private double merge() throws IOException { - - int ln = 50; - int dn = 300; - Set<String> testSet = new HashSet<String>(); - HyperLogLogPlusCounter[] hllcs = new HyperLogLogPlusCounter[ln]; - for (int i = 0; i < ln; i++) { - hllcs[i] = newHLLC(); - for (int k = 0; k < dn; k++) { - String[] samples = generateSampleData(); - for (String data : samples) { - testSet.add(data); - hllcs[i].add(Bytes.toBytes(data)); - } - } - } - HyperLogLogPlusCounter mergeHllc = newHLLC(); - for (HyperLogLogPlusCounter hllc : hllcs) { - mergeHllc.merge(hllc); - checkSerialize(mergeHllc); - } - - double errorRate = mergeHllc.getErrorRate(); - long estimate = mergeHllc.getCountEstimate(); - double actualError = (double) (testSet.size() - estimate) / testSet.size(); - - System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError); - - if (Math.abs(actualError) > errorRate) { - errorCount1++; - } - if (Math.abs(actualError) > 2 * errorRate) { - errorCount2++; - } - if (Math.abs(actualError) > 3 * errorRate) { - errorCount3++; - } - - return actualError; - } - - @Test - public void testPerformance() throws IOException { - int N = 3; // reduce N HLLC into one - int M = 1000; // for M times, use 100000 for real perf test - - HyperLogLogPlusCounter samples[] = new HyperLogLogPlusCounter[N]; - for (int i = 0; i < N; i++) { - samples[i] = newHLLC(); - for (String str : generateTestData(10000)) - samples[i].add(str); - } - - System.out.println("Perf test running ... "); - long start = System.currentTimeMillis(); - HyperLogLogPlusCounter sum = newHLLC(); - for (int i = 0; i < M; i++) { - sum.clear(); - for (int j = 0; j < N; j++) { - sum.merge(samples[j]); - checkSerialize(sum); - } - } - long duration = System.currentTimeMillis() - start; - System.out.println("Perf test result: " + duration / 1000 + " seconds"); - } - - @Test - public void testEquivalence() { - byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 }; - byte[] b = new byte[] { 3, 4, 42 }; - HyperLogLogPlusCounter ha = new HyperLogLogPlusCounter(); - HyperLogLogPlusCounter hb = new HyperLogLogPlusCounter(); - ha.add(a, 1, 3); - hb.add(b); - - Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate()); - } - - private HyperLogLogPlusCounter newHLLC() { - return new HyperLogLogPlusCounter(16); - } -}
