# Deleted IGFS load tests as they are no longer used.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/ffcf04ca Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/ffcf04ca Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/ffcf04ca Branch: refs/heads/ignite-51 Commit: ffcf04caf38d0a303707ff1a8d846b28ae8fbd1a Parents: e520021 Author: vozerov-gridgain <voze...@gridgain.com> Authored: Tue Mar 10 11:44:28 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Tue Mar 10 11:44:28 2015 +0300 ---------------------------------------------------------------------- .../ignite/loadtests/igfs/IgfsNodeStartup.java | 48 ---- .../igfs/IgfsPerformanceBenchmark.java | 281 ------------------- 2 files changed, 329 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/ffcf04ca/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsNodeStartup.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsNodeStartup.java b/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsNodeStartup.java deleted file mode 100644 index e4cffbe..0000000 --- a/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsNodeStartup.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.loadtests.igfs; - -import org.apache.ignite.*; -import org.apache.ignite.internal.util.typedef.*; - -import javax.swing.*; - -/** - * Node startup for IGFS performance benchmark. - */ -public class IgfsNodeStartup { - /** - * Start up an empty node with specified cache configuration. - * - * @param args Command line arguments, none required. - */ - public static void main(String[] args) { - try (Ignite ignored = G.start("config/hadoop/default-config.xml")) { - // Wait until Ok is pressed. - JOptionPane.showMessageDialog( - null, - new JComponent[] { - new JLabel("Ignite started."), - new JLabel("Press OK to stop Ignite.") - }, - "Ignite", - JOptionPane.INFORMATION_MESSAGE - ); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/ffcf04ca/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsPerformanceBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsPerformanceBenchmark.java b/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsPerformanceBenchmark.java deleted file mode 100644 index 00dc4f5..0000000 --- a/modules/hadoop/src/test/java/org/apache/ignite/loadtests/igfs/IgfsPerformanceBenchmark.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.loadtests.igfs; - -import org.apache.hadoop.conf.*; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.fs.FileSystem; -import org.apache.ignite.internal.*; -import org.apache.ignite.internal.util.typedef.internal.*; -import org.apache.ignite.testframework.*; -import org.jetbrains.annotations.*; - -import java.io.*; -import java.util.*; -import java.util.concurrent.atomic.*; - -/** - * Tests write throughput. - */ -public class IgfsPerformanceBenchmark { - /** Path to test hadoop configuration. */ - private static final String HADOOP_FS_CFG = "modules/core/src/test/config/hadoop/core-site.xml"; - - /** FS prefix. */ - private static final String FS_PREFIX = "igfs:///"; - - /** Test writes. */ - private static final int OP_WRITE = 0; - - /** Test reads. */ - private static final int OP_READ = 1; - - /** - * Starts benchmark. - * - * @param args Program arguments. - * [0] - number of threads, default 1. - * [1] - file length, default is 1GB. - * [2] - stream buffer size, default is 1M. - * [3] - fs config path. - * @throws Exception If failed. - */ - public static void main(String[] args) throws Exception { - final int threadNum = intArgument(args, 0, 1); - final int op = intArgument(args, 1, OP_WRITE); - final long fileLen = longArgument(args, 2, 256 * 1024 * 1024); - final int bufSize = intArgument(args, 3, 128 * 1024); - final String cfgPath = argument(args, 4, HADOOP_FS_CFG); - final String fsPrefix = argument(args, 5, FS_PREFIX); - final short replication = (short)intArgument(args, 6, 3); - - final Path igfsHome = new Path(fsPrefix); - - final FileSystem fs = igfs(igfsHome, cfgPath); - - final AtomicLong progress = new AtomicLong(); - - final AtomicInteger idx = new AtomicInteger(); - - System.out.println("Warming up..."); - -// warmUp(fs, igfsHome, op, fileLen); - - System.out.println("Finished warm up."); - - if (op == OP_READ) { - for (int i = 0; i < threadNum; i++) - benchmarkWrite(fs, new Path(igfsHome, "in-" + i), fileLen, bufSize, replication, null); - } - - long total = 0; - - long start = System.currentTimeMillis(); - - IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() { - @Override public void run() { - String fileIdx = op == OP_READ ? String.valueOf(idx.getAndIncrement()) : UUID.randomUUID().toString(); - - try { - for (int i = 0; i < 200; i++) { - if (op == OP_WRITE) - benchmarkWrite(fs, new Path(igfsHome, "out-" + fileIdx), fileLen, bufSize, replication, - progress); - else - benchmarkRead(fs, new Path(igfsHome, "in-" + fileIdx), bufSize, progress); - } - - System.out.println("Finished " + (op == OP_WRITE ? "writing" : "reading") + " data."); - } - catch (Exception e) { - System.out.println("Failed to process stream: " + e); - - e.printStackTrace(); - } - } - }, threadNum, "test-runner"); - - while (!fut.isDone()) { - U.sleep(1000); - - long written = progress.getAndSet(0); - - total += written; - - int mbytesPerSec = (int)(written / (1024 * 1024)); - - System.out.println((op == OP_WRITE ? "Write" : "Read") + " rate [threads=" + threadNum + - ", bufSize=" + bufSize + ", MBytes/s=" + mbytesPerSec + ']'); - } - - long now = System.currentTimeMillis(); - - System.out.println((op == OP_WRITE ? "Written" : "Read") + " " + total + " bytes in " + (now - start) + - "ms, avg write rate is " + (total * 1000 / ((now - start) * 1024 * 1024)) + "MBytes/s"); - - fs.close(); - } - - /** - * Warms up server side. - * - * @param fs File system. - * @param igfsHome IGFS home. - * @throws Exception If failed. - */ - private static void warmUp(FileSystem fs, Path igfsHome, int op, long fileLen) throws Exception { - Path file = new Path(igfsHome, "out-0"); - - benchmarkWrite(fs, file, fileLen, 1024 * 1024, (short)1, null); - - for (int i = 0; i < 5; i++) { - if (op == OP_WRITE) - benchmarkWrite(fs, file, fileLen, 1024 * 1024, (short)1, null); - else - benchmarkRead(fs, file, 1024 * 1024, null); - } - - fs.delete(file, true); - } - - /** - * @param args Arguments. - * @param idx Index. - * @param dflt Default value. - * @return Argument value. - */ - private static int intArgument(String[] args, int idx, int dflt) { - if (args.length <= idx) - return dflt; - - try { - return Integer.parseInt(args[idx]); - } - catch (NumberFormatException ignored) { - return dflt; - } - } - - /** - * @param args Arguments. - * @param idx Index. - * @param dflt Default value. - * @return Argument value. - */ - private static long longArgument(String[] args, int idx, long dflt) { - if (args.length <= idx) - return dflt; - - try { - return Long.parseLong(args[idx]); - } - catch (NumberFormatException ignored) { - return dflt; - } - } - - /** - * @param args Arguments. - * @param idx Index. - * @param dflt Default value. - * @return Argument value. - */ - private static String argument(String[] args, int idx, String dflt) { - if (args.length <= idx) - return dflt; - - return args[idx]; - } - - /** - * Get IGFS FileSystem. - * - * @param home Home path. - * @param cfgPath Config path. - * @return FileSystem. - * @throws IOException If failed. - */ - private static FileSystem igfs(Path home, String cfgPath) throws IOException { - Configuration cfg = new Configuration(); - - cfg.addResource(U.resolveIgniteUrl(cfgPath)); - - return FileSystem.get(home.toUri(), cfg); - } - - /** - * Tests stream write to specified file. - * - * @param file File to write to. - * @param len Length to write. - * @param bufSize Buffer size. - * @param replication Replication factor. - * @param progress Progress that will be incremented on each written chunk. - */ - private static void benchmarkWrite(FileSystem fs, Path file, long len, int bufSize, short replication, - @Nullable AtomicLong progress) throws Exception { - - try (FSDataOutputStream out = fs.create(file, true, bufSize, replication, fs.getDefaultBlockSize())) { - long written = 0; - - byte[] data = new byte[bufSize]; - - while (written < len) { - int chunk = (int) Math.min(len - written, bufSize); - - out.write(data, 0, chunk); - - written += chunk; - - if (progress != null) - progress.addAndGet(chunk); - } - - out.flush(); - } - catch (Exception e) { - e.printStackTrace(); - throw e; - } - } - - /** - * Tests stream read from specified file. - * - * @param file File to write from - * @param bufSize Buffer size. - * @param progress Progress that will be incremented on each written chunk. - */ - private static void benchmarkRead(FileSystem fs, Path file, int bufSize, @Nullable AtomicLong progress) - throws Exception { - - try (FSDataInputStream in = fs.open(file, bufSize)) { - byte[] data = new byte[32 * bufSize]; - - while (true) { - int read = in.read(data); - - if (read < 0) - return; - - if (progress != null) - progress.addAndGet(read); - } - } - } -}
