http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/bench/lib/tservers.py ---------------------------------------------------------------------- diff --git a/test/system/bench/lib/tservers.py b/test/system/bench/lib/tservers.py deleted file mode 100755 index b34397a..0000000 --- a/test/system/bench/lib/tservers.py +++ /dev/null @@ -1,89 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import fcntl - -import os -import time -import select -import subprocess - -from lib.path import accumuloConf -from lib.options import log - -def tserverNames(): - return [s.strip() for s in open(accumuloConf('tservers'))] - -def runEach(commandMap): - result = {} - handles = [] - for tserver, command in commandMap.items(): - log.debug("ssh: %s: %s", tserver, command) - handle = subprocess.Popen(['ssh',tserver] + [command], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - for h in handle.stdout, handle.stderr: - fcntl.fcntl(h, fcntl.F_SETFL, os.O_NDELAY) - handle.tserver = tserver - handle.command = command - handle.start = time.time() - handles.append(handle) - handlesLeft = set(handles[:]) - while handlesLeft: - fds = {} - doomed = set() - for h in handlesLeft: - more = [] - if h.stdout != None: - more.append(h.stdout) - if h.stderr != None: - more.append(h.stderr) - for m in more: - fds[m] = h - if not more: - doomed.add(h) - handlesLeft -= doomed - if not handlesLeft: break - rd, wr, ex = select.select(fds.keys(), [], [], 10) - for r in rd: - handle = fds[r] - data = r.read(1024) - result.setdefault(handle, ['', '']) - if not data: - if r == handle.stdout: - handle.stdout = None - else: - handle.stderr = None - if r == handle.stdout: - result[handle][0] += data - else: - result[handle][1] += data - if handle.stdout == None and handle.stderr == None: - log.debug("Tserver %s finished in %.2f", - handle.tserver, - time.time() - handle.start) - handle.wait() - if not rd: - log.debug("Waiting on %d tservers (%s...)", - len(handlesLeft), - ', '.join([h.tserver for h in handlesLeft])[:50]) - return dict([(h.tserver, (h.returncode, out, err)) - for h, (out, err) in result.items()]) - -def runAll(command): - tservers = tserverNames() - log.debug("Running %s on %s..", command, ', '.join(tservers)[:50]) - return runEach(dict([(s, command) for s in tservers])) -
http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/bench/lib/util.py ---------------------------------------------------------------------- diff --git a/test/system/bench/lib/util.py b/test/system/bench/lib/util.py deleted file mode 100755 index bfe38db..0000000 --- a/test/system/bench/lib/util.py +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - - -def sleep(n): - time.sleep(n) http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/bench/run.py ---------------------------------------------------------------------- diff --git a/test/system/bench/run.py b/test/system/bench/run.py deleted file mode 100755 index 63a21fa..0000000 --- a/test/system/bench/run.py +++ /dev/null @@ -1,116 +0,0 @@ -#! /usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import getopt -import os -import sys -import logging -import unittest - -from lib.options import options, args, log -from lib.Benchmark import Benchmark - -def getBenchmarks(): - import glob - result = [] - here = os.path.dirname(__file__) - sys.path.insert(0, here) - for path in glob.glob('%s/*/*.py' % here): - path = path[len(here):] - if path.find('__init__') >= 0: continue - if path.find('/lib/') >= 0: continue - moduleName = path.replace(os.path.sep, '.') - moduleName = moduleName.lstrip('.')[:-3] - module = __import__(moduleName, globals(), locals(), [moduleName]) - result.extend(list(module.suite())) - return result - -def benchComparator(first, second): - if (first.name() < second.name()): - return -1 - elif (second.name() < first.name()): - return 1 - else: - return 0 - -def main(): - if not os.getenv('HADOOP_HOME'): - print 'Please set the environment variable \'HADOOP_HOME\' before running the benchmarks' - sys.exit(0) - if not os.getenv('ZOOKEEPER_HOME'): - print 'Please set the environment variable \'ZOOKEEPER_HOME\' before running the benchmarks' - sys.exit(0) - if not os.getenv('ACCUMULO_HOME'): - print 'Please set the environment variable \'ACCUMULO_HOME\' before running the benchmarks' - sys.exit(0) - import textwrap - benchmarks = getBenchmarks() - benchmarks.sort(benchComparator) - auth = 0 - for b in benchmarks: - b.setSpeed(options.runSpeed) - if auth == 0 and b.needsAuthentication > 0: - auth = 1 - if options.list: - indent = len(benchmarks[0].name()) - wrap = 78 - indent - prefix = ' ' * indent + ' ' - for b in benchmarks: - desc = b.shortDescription() or "No description" - desc = textwrap.wrap(desc, wrap) - desc = '\n'.join([(prefix + line) for line in desc]) - print '%*s: %s' % (indent, b.name(), desc.lstrip()) - sys.exit(0) - logging.basicConfig(level=options.logLevel) - if auth == 1: - if options.user == '': - print 'User: ', - user = sys.stdin.readline().strip() - else: - user = options.user - if options.password == '': - import getpass - password = getpass.getpass('Password: ') - else: - password = options.password - if options.zookeepers == '': - print 'Zookeepers: ', - zookeepers = sys.stdin.readline().strip() - else: - zookeepers = options.zookeepers - if options.instance == '': - print 'Instance: ', - instance = sys.stdin.readline().strip() - else: - instance = options.instance - Benchmark.instance = instance - Benchmark.zookeepers = zookeepers - Benchmark.instance = instance - Benchmark.password = password - Benchmark.username = user - if args: - benchmarks = [ - b for b in benchmarks if b.name() in args - ] - runner = unittest.TextTestRunner(verbosity=2) - runner.run(unittest.TestSuite(benchmarks)) - for b in benchmarks: - log.info("%30s: %5.2f", b.name(), b.score()) - -if __name__ == '__main__': - main() http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/merkle-replication/README ---------------------------------------------------------------------- diff --git a/test/system/merkle-replication/README b/test/system/merkle-replication/README deleted file mode 100644 index b892491..0000000 --- a/test/system/merkle-replication/README +++ /dev/null @@ -1,65 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Distributed Replication Test -=========================== - -Ingests random data into a table configured for replication, and then -verifies that the original table and the replicated table have equivalent -data using a Merkle tree. - -* Steps to run test - -1. Configure merkle-env.sh - - Be sure to set the username/password to connect to Accumulo and - the Accumulo instance name and ZooKeepers. The example defaults - to using the same instance as the source and destination. - - The randomness of the generated data can be controlled via some - parameters in this file. Note that if deletes are introduced, it - is very likely to cause an incorrect verification since the tombstone'ing - of that delete will differ on the source and remote due to the order - of minor compactions that occur. - -2. Run configure-replication.sh - - This script sets up the two instances for replication, creates - the tables that will be replicated and adds some splits to them. - - This is destructive to any existing replication configuration so - use it with care on a real instance. - -3. Run ingest-data.sh - - Ingests the configured amount of random data into the source - table. - -4. Run 'accumulo-cluster stop' && 'accumulo-cluster start' on the source instance - - A tabletserver in the source instance is likely to still be referencing - a WAL for a presently online tablet which will prevent that - file from being replicated. Stopping the local instance will ensure - that all WALs are candidate for replication. - -5. Run verify-data.sh - - This will compute the leaves merkle tree for the source and destination - tables and then compute the root hash for both. The root hash - is presented to the user. - - If the root hashes are equal, the test passed; otherwise, the test fails. http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/merkle-replication/configure-replication.sh ---------------------------------------------------------------------- diff --git a/test/system/merkle-replication/configure-replication.sh b/test/system/merkle-replication/configure-replication.sh deleted file mode 100755 index 44ebdd7..0000000 --- a/test/system/merkle-replication/configure-replication.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink - dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${dir}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -script=$( basename "${SOURCE}" ) -# Stop: Resolve Script Directory - -# Guess at ACCUMULO_HOME and ACCUMULO_CONF_DIR if not already defined -ACCUMULO_HOME=${ACCUMULO_HOME:-"${dir}/../../.."} -ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME/conf"} - -# Get the configuration values -. ./merkle-env.sh - -tmpdir=$(mktemp -dt "$0.XXXXXXXXXX") - -source_commands="${tmpdir}/source_commands.txt" - -echo 'Removing old tables and setting replication name on source' - -echo "deletetable -f $SOURCE_TABLE_NAME" >> $source_commands -echo "createtable $SOURCE_TABLE_NAME" >> $source_commands -echo "config -s replication.name=source" >> $source_commands -echo "quit" >> $source_commands - -# Source: drop and create tables, configure unique name for replication and grant perms -echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \ - $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -f $source_commands - -destination_commands="${tmpdir}/destination_commands.txt" - -echo 'Removing old tables and setting replication name on destination' - -echo "deletetable -f $DESTINATION_TABLE_NAME" >> $destination_commands -echo "createtable $DESTINATION_TABLE_NAME" >> $destination_commands -echo "config -s replication.name=destination" >> $destination_commands -echo "quit" >> $destination_commands - -# Destination: drop and create tables, configure unique name for replication and grant perms -echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \ - $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -f $destination_commands - -rm $source_commands -rm $destination_commands - -table_id=$(echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \ - $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -e 'tables -l' | grep "${DESTINATION_TABLE_NAME}" \ - | grep -v "${DESTINATION_MERKLE_TABLE_NAME}" | awk '{print $3}') - -echo "Configuring $SOURCE_TABLE_NAME to replicate to $DESTINATION_TABLE_NAME (id=$table_id)" - -# Define our peer 'destination' with the ReplicaSystem impl, instance name and ZKs -echo "config -s replication.peer.destination=org.apache.accumulo.tserver.replication.AccumuloReplicaSystem,$DESTINATION_INSTANCE,$DESTINATION_ZOOKEEPERS" >> $source_commands -# Username for 'destination' -echo "config -s replication.peer.user.destination=$DESTINATION_ACCUMULO_USER" >> $source_commands -# Password for 'destination' -echo "config -s replication.peer.password.destination=$DESTINATION_ACCUMULO_PASSWORD" >> $source_commands -# Configure replication to 'destination' for $SOURCE_TABLE_NAME -echo "config -t $SOURCE_TABLE_NAME -s table.replication.target.destination=$table_id" >> $source_commands -# Enable replication for the table -echo "config -t $SOURCE_TABLE_NAME -s table.replication=true" >> $source_commands -echo "quit" >> $source_commands - -# Configure replication from source to destination and then enable it -echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \ - $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -f $source_commands - -rm $source_commands - -# Add some splits to make ingest faster -echo 'Adding splits...' - -echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \ - $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -e "addsplits -t $SOURCE_TABLE_NAME 1 2 3 4 5 6 7 8 9" - -echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \ - $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -e "addsplits -t $DESTINATION_TABLE_NAME 1 2 3 4 5 6 7 8 9" - http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/merkle-replication/ingest-data.sh ---------------------------------------------------------------------- diff --git a/test/system/merkle-replication/ingest-data.sh b/test/system/merkle-replication/ingest-data.sh deleted file mode 100755 index 91b8ccc..0000000 --- a/test/system/merkle-replication/ingest-data.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink - dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${dir}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -script=$( basename "${SOURCE}" ) -# Stop: Resolve Script Directory - -# Guess at ACCUMULO_HOME and ACCUMULO_CONF_DIR if not already defined -ACCUMULO_HOME=${ACCUMULO_HOME:-"${dir}/../../.."} -ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME/conf"} - -# Get the configuration values -. ./merkle-env.sh - -# Ingest data into the source table -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.ingest.RandomWorkload --table $SOURCE_TABLE_NAME \ - -i $SOURCE_INSTANCE -z $SOURCE_ZOOKEEPERS -u $SOURCE_ACCUMULO_USER -p $SOURCE_ACCUMULO_PASSWORD -d $DELETE_PERCENT \ - -cf $MAX_CF -cq $MAX_CQ -r $MAX_ROW -n $NUM_RECORDS http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/merkle-replication/merkle-env.sh ---------------------------------------------------------------------- diff --git a/test/system/merkle-replication/merkle-env.sh b/test/system/merkle-replication/merkle-env.sh deleted file mode 100755 index d405394..0000000 --- a/test/system/merkle-replication/merkle-env.sh +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Random data will be written to this table -SOURCE_TABLE_NAME='replicationSource' -# Then replicated to this table -DESTINATION_TABLE_NAME='replicationDestination' - -# Merkle tree to be stored in this table for the source table -SOURCE_MERKLE_TABLE_NAME="${SOURCE_TABLE_NAME}_merkle" -# Merkle tree to be stored in this table for the destination table -DESTINATION_MERKLE_TABLE_NAME="${DESTINATION_TABLE_NAME}_merkle" - -# Connection information to Accumulo -SOURCE_ACCUMULO_USER="user" -SOURCE_ACCUMULO_PASSWORD="password" - -DESTINATION_ACCUMULO_USER="${SOURCE_ACCUMULO_USER}" -DESTINATION_ACCUMULO_PASSWORD="${SOURCE_ACCUMULO_PASSWORD}" - -SOURCE_INSTANCE="accumulo" -DESTINATION_INSTANCE="${SOURCE_INSTANCE}" - -SOURCE_ZOOKEEPERS="localhost" -DESTINATION_ZOOKEEPERS="${SOURCE_ZOOKEEPERS}" - -# Accumulo user to be configured on the destination instance -#REPLICATION_USER="${ACCUMULO_USER}" -#REPLICATION_PASSWORD="${ACCUMULO_PASSWORD}" - -# Control amount and distribution of data written -NUM_RECORDS=100000000 -MAX_ROW=1000000 -MAX_CF=10 -MAX_CQ=100 -DELETE_PERCENT=0 http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/merkle-replication/verify-data.sh ---------------------------------------------------------------------- diff --git a/test/system/merkle-replication/verify-data.sh b/test/system/merkle-replication/verify-data.sh deleted file mode 100755 index 225d892..0000000 --- a/test/system/merkle-replication/verify-data.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink - dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${dir}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -script=$( basename "${SOURCE}" ) -# Stop: Resolve Script Directory - -# Guess at ACCUMULO_HOME and ACCUMULO_CONF_DIR if not already defined -ACCUMULO_HOME=${ACCUMULO_HOME:-"${dir}/../../.."} -ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME/conf"} - -# Get the configuration values -. ./merkle-env.sh - -tmpdir=$(mktemp -dt "$0.XXXXXXXXXX") - -splits=${tmpdir}/splits - -echo 1 >> $splits -echo 2 >> $splits -echo 3 >> $splits -echo 4 >> $splits -echo 5 >> $splits -echo 6 >> $splits -echo 7 >> $splits -echo 8 >> $splits -echo 9 >> $splits - -commands=${tmpdir}/commands - -# Generate leaves of merkle trees for source -echo "deletetable -f $SOURCE_MERKLE_TABLE_NAME" >> $commands -echo "createtable $SOURCE_MERKLE_TABLE_NAME" >> $commands -echo "quit" >> $commands - -echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \ - $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -f $commands - -echo -e "\nGenerating merkle tree hashes for $SOURCE_TABLE_NAME" - -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.GenerateHashes -t $SOURCE_TABLE_NAME \ - -o $SOURCE_MERKLE_TABLE_NAME -i $SOURCE_INSTANCE -z $SOURCE_ZOOKEEPERS -u $SOURCE_ACCUMULO_USER \ - -p $SOURCE_ACCUMULO_PASSWORD -nt 8 -hash MD5 --splits $splits - -rm $commands - -# Generate leaves of merkle trees for destination -echo "deletetable -f $DESTINATION_MERKLE_TABLE_NAME" >> $commands -echo "createtable $DESTINATION_MERKLE_TABLE_NAME" >> $commands -echo "quit" >> $commands - -echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \ - $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -f $commands - -echo -e "\nGenerating merkle tree hashes for $DESTINATION_TABLE_NAME" - -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.GenerateHashes -t $DESTINATION_TABLE_NAME \ - -o $DESTINATION_MERKLE_TABLE_NAME -i $DESTINATION_INSTANCE -z $DESTINATION_ZOOKEEPERS -u $DESTINATION_ACCUMULO_USER \ - -p $DESTINATION_ACCUMULO_PASSWORD -nt 8 -hash MD5 --splits $splits - -echo -e "\nComputing root hash:" - -#Compute root node of merkle tree -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.ComputeRootHash -t $SOURCE_MERKLE_TABLE_NAME \ - -i $SOURCE_INSTANCE -z $SOURCE_ZOOKEEPERS -u $SOURCE_ACCUMULO_USER -p $SOURCE_ACCUMULO_PASSWORD -hash MD5 - -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.ComputeRootHash -t $DESTINATION_MERKLE_TABLE_NAME \ - -i $DESTINATION_INSTANCE -z $DESTINATION_ZOOKEEPERS -u $DESTINATION_ACCUMULO_USER -p $DESTINATION_ACCUMULO_PASSWORD -hash MD5 - -rm -rf $tmpdir http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/scalability/README.md ---------------------------------------------------------------------- diff --git a/test/system/scalability/README.md b/test/system/scalability/README.md deleted file mode 100644 index 4f287d2..0000000 --- a/test/system/scalability/README.md +++ /dev/null @@ -1,57 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Apache Accumulo Scalability Tests -================================= - -The scalability test framework needs to be configured for your Accumulo -instance by performing the following steps. - -WARNING: Each scalability test rewrites your `conf/tservers` file and reinitializes -your Accumulo instance. Do not run these tests on a cluster holding essential -data. - -1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your - `$ACCUMULO_CONF_DIR/accumulo-env.sh.` - -2. Create a 'site.conf' file in the `conf` directory containing settings - needed by test nodes to connect to Accumulo, and to guide the tests. - - `$ cp conf/site.conf.example conf/site.conf` - -3. Create an 'Ingest.conf' file in the `conf` directory containing performance - settings for the Ingest test. (This test is currently the only scalability - test available.) - - `$ cp conf/Ingest.conf.example conf/Ingest.conf` - - Each test has a unique ID (e.g., "Ingest") which correlates with its test - code in: - - `org.apache.accumulo.test.scalability.tests.<ID>` - - This ID correlates with a config file: - - `conf/<ID>.conf` - -To run the test, specify its ID to the run.py script. - -> `$ nohup ./run.py Ingest > test1.log 2>&1 &` - -A timestamped directory will be created, and results are placed in it as each -test completes. - http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/scalability/conf/Ingest.conf.example ---------------------------------------------------------------------- diff --git a/test/system/scalability/conf/Ingest.conf.example b/test/system/scalability/conf/Ingest.conf.example deleted file mode 100644 index 38800a1..0000000 --- a/test/system/scalability/conf/Ingest.conf.example +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Table to ingest into -TABLE=ingest -# Number of entries to ingest -NUM_ENTRIES=3000000 - -# Batch writer configuration -# Max memory (in bytes) that each ingester will use to buffer writes -MAX_MEMORY=100000000 -# Max time (in millis) that each ingester will buffer data -MAX_LATENCY=600000 -# Number of threads each ingester will use to write data -NUM_THREADS=4 http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/scalability/conf/site.conf.example ---------------------------------------------------------------------- diff --git a/test/system/scalability/conf/site.conf.example b/test/system/scalability/conf/site.conf.example deleted file mode 100644 index e1ce08f..0000000 --- a/test/system/scalability/conf/site.conf.example +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# accumulo instance -INSTANCE_NAME=a12 -# zookeepers -ZOOKEEPERS=localhost -# username -USER=root -# password -PASSWORD=secret -# used to determine which nodes to run tests on -TSERVERS=/accumulo/conf/tservers -# sets the number of nodes for each test case -TEST_CASES=1,2,4,8,16 http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/scalability/run.py ---------------------------------------------------------------------- diff --git a/test/system/scalability/run.py b/test/system/scalability/run.py deleted file mode 100755 index 2b21a12..0000000 --- a/test/system/scalability/run.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging -import time -import os -import sys -from ConfigParser import ConfigParser -from subprocess import Popen, PIPE - -class JavaConfig: - '''Enable access to properities in java siteConfig file''' - def __init__(self, fname): - self.prop_d = {} - for line in open(fname): - line = line.strip(); - if line.startswith('#') or len(line) == 0: - continue - pair = line.split('=') - if len(pair) != 2: - log.error("Invalid property (%s)" % line) - continue - self.prop_d[pair[0].strip()] = pair[1].strip() - - def get(self, prop): - return self.prop_d[prop] - -def file_len(fname): - i=0 - for line in open(fname): - i += 1 - return i - -def runTest(testName, siteConfig, testDir, numNodes, fdata): - - log('Stopping accumulo') - syscall('$ACCUMULO_HOME/bin/accumulo-cluster stop') - - log('Creating tservers file for this test') - tserversPath = siteConfig.get('TSERVERS') - nodesPath = testDir+'/nodes/%d' % numNodes - syscall('head -n %d %s > %s' % (numNodes,tserversPath,nodesPath)) - - log('Copying tservers file to accumulo config') - syscall('cp '+nodesPath+' $ACCUMULO_CONF_DIR/tservers'); - - log('Removing /accumulo directory in HDFS') - syscall("hadoop fs -rmr /accumulo") - - log('Initializing new Accumulo instance') - instance = siteConfig.get('INSTANCE_NAME') - passwd = siteConfig.get('PASSWORD') - syscall('printf "%s\nY\n%s\n%s\n" | $ACCUMULO_HOME/bin/accumulo init' % (instance, passwd, passwd)) - - log('Starting new Accumulo instance') - syscall('$ACCUMULO_HOME/bin/accumulo-cluster start') - - sleepTime = 30 - if numNodes > 120: - sleepTime = int(numNodes / 4) - log('Sleeping for %d seconds' % sleepTime) - time.sleep(sleepTime) - - log('Setting up %s test' % testName) - syscall('$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.scalability.Run %s setup %s' % (testName, numNodes)) - - log('Sleeping for 5 seconds') - time.sleep(5) - - log('Starting %s clients' % testName) - numThreads = numNodes - if int(numNodes) > 128: - numThreads='128' - syscall('pssh -P -h %s -p %s "$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.scalability.Run %s client %s >/tmp/scale.out 2>/tmp/scale.err &" < /dev/null' % (nodesPath, numThreads, testName, numNodes)) - - log('Sleeping for 30 sec before checking how many clients started...') - time.sleep(30) - output = Popen(["hadoop fs -ls /accumulo-scale/clients"], stdout=PIPE, shell=True).communicate()[0] - num_clients = int(output.split()[1]) - log('%s clients started!' % num_clients) - - log('Waiting until %d clients finish.' % num_clients) - last = 0 - done = 0 - while done < num_clients: - time.sleep(5) - output = Popen(["hadoop fs -ls /accumulo-scale/results"], stdout=PIPE, shell=True).communicate()[0] - if not output: - sys.stdout.write('.') - sys.stdout.flush() - continue - done = int(output.split()[1]) - if done != last: - sys.stdout.write('.%s' % done) - else: - sys.stdout.write('.') - sys.stdout.flush() - last = done - sys.stdout.flush() - log('\nAll clients are finished!') - - log('Copying results from HDFS') - resultsDir = "%s/results/%s" % (testDir, numNodes) - syscall('hadoop fs -copyToLocal /accumulo-scale/results %s' % resultsDir) - - log('Calculating results from clients') - times = [] - totalMs = 0L - totalEntries = 0L - totalBytes = 0L - for fn in os.listdir(resultsDir): - for line in open('%s/%s' % (resultsDir,fn)): - words = line.split() - if words[0] == 'ELAPSEDMS': - ms = long(words[1].strip()) - totalMs += ms - times.append(ms) - totalEntries += long(words[2].strip()) - totalBytes += long(words[3].strip()) - times.sort() - - print times - numClients = len(times) - min = times[0] / 1000 - avg = (float(totalMs) / numClients) / 1000 - median = times[int(numClients/2)] / 1000 - max = times[numClients-1] / 1000 - q1 = times[int(numClients/4)] / 1000 - q3 = times[int((3*numClients)/4)] / 1000 - - log('Tservs\tClients\tMin\tQ1\tAvg\tMed\tQ3\tMax\tEntries\tMB') - resultStr = '%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%dM\t%d' % (numNodes, numClients, min, q1, avg, median, q3, max, totalEntries / 1000000, totalBytes / 1000000) - log(resultStr) - fdata.write(resultStr + '\n') - fdata.flush() - - time.sleep(5) - - log('Tearing down %s test' % testName) - syscall('$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.scalability.Run %s teardown %s' % (testName, numNodes)) - - time.sleep(10) - -def syscall(cmd): - log('> %s' % cmd) - os.system(cmd) - -def run(cmd, **kwargs): - log.debug("Running %s", ' '.join(cmd)) - handle = Popen(cmd, stdout=PIPE, **kwargs) - out, err = handle.communicate() - log.debug("Result %d (%r, %r)", handle.returncode, out, err) - return handle.returncode - -def log(msg): - print msg - sys.stdout.flush() - -def main(): - - if not os.getenv('ACCUMULO_HOME'): - raise 'ACCUMULO_HOME needs to be set!' - - if not os.getenv('ACCUMULO_CONF_DIR'): - raise 'ACCUMULO_CONF_DIR needs to be set!' - - if not os.getenv('HADOOP_HOME'): - raise 'HADOOP_HOME needs to be set!' - - if len(sys.argv) != 2: - log('Usage: run.py <testName>') - sys.exit() - testName = sys.argv[1] - - logging.basicConfig(level=logging.DEBUG) - - log('Creating test directory structure') - testDir = 'test-%d' % time.time() - nodesDir = testDir+'/nodes' - resultsDir = testDir+'/results' - syscall('mkdir %s' % testDir) - syscall('mkdir %s' % nodesDir) - syscall('mkdir %s' % resultsDir) - - log('Removing current /accumulo-scale directory') - syscall('hadoop fs -rmr /accumulo-scale') - - log('Creating new /accumulo-scale directory structure') - syscall('hadoop fs -mkdir /accumulo-scale') - syscall('hadoop fs -mkdir /accumulo-scale/clients') - syscall('hadoop fs -mkdir /accumulo-scale/results') - syscall('hadoop fs -chmod -R 777 /accumulo-scale') - - log('Copying config to HDFS') - syscall('hadoop fs -copyFromLocal ./conf /accumulo-scale/conf') - - siteConfig = JavaConfig('conf/site.conf'); - tserversPath = siteConfig.get('TSERVERS') - maxNodes = file_len(tserversPath) - - fdata = open('%s/scale.dat' % testDir, 'w') - fdata.write('Tservs\tClients\tMin\tAvg\tMed\tMax\tEntries\tMB\n') - - for numNodes in siteConfig.get('TEST_CASES').split(','): - log('Running %s test with %s nodes' % (testName, numNodes)) - if int(numNodes) > maxNodes: - logging.error('Skipping %r test case as tservers file %r contains only %r nodes', numNodes, tserversPath, maxNodes) - continue - runTest(testName, siteConfig, testDir, int(numNodes), fdata) - sys.stdout.flush() - -if __name__ == '__main__': - main() http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/README.md ---------------------------------------------------------------------- diff --git a/test/system/stress/README.md b/test/system/stress/README.md deleted file mode 100644 index 9eb9fdc..0000000 --- a/test/system/stress/README.md +++ /dev/null @@ -1,105 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Accumulo Stress Test -==================== - -This is an attempt to observe the behavior Accumulo displays when compacting and -reading cells. There are two components to this package: -`org.apache.accumulo.test.stress.random.Write` and -`org.apache.accumulo.test.stress.random.Scan`. - -Since this test is meant to stress the system, there is no pre-defined test -period and no success criteria that we can programmatically check. During a -successful test period, the Accumulo cluster should remain stable and -functional. Possible failure conditions include component failures, such as -tablet servers running out of memory. - -The `org.apache.accumulo.test.stress.random.Write` utility provides facilities -for writing random sized cells. Users can configure minimum and maximum sized -portions of a cell. The portions users can configure are the row, column family, -column qualifier and value. Note that the sizes are uniformly distributed -between the minimum and maximum values. - -The `org.apache.accumulo.test.stress.random.Scan` utility provides users with -the ability to query tables generated by the Write. It will pick a tablet at -random and scan the entire range. The amount of times this process is performed -is user configurable. By default, it happens 1,024 times. Users can also specify -whether or not the scan should be isolated. - -There is no shared state intended by either of these services. This allows -multiple clients to be run in parallel, either on the same host or distributed -across hosts. - -## Prerequisites - -The utilities are run on a normal, initialized Accumulo cluster. By default, -they only work with a table named "stress_test". - -The start and stop scripts rely on pssh. Before running them, you will need -to install pssh on the machines that will be controlled by them. - -## Running - -Copy `stress-env.sh.example` to `stress-env.sh` and edit it, supplying at least -correct values for the USERPASS and INSTANCE variables. The remainder of this -file describes and provides defaults for many of the configuration options -for the stress utilities. - -### Individual Execution - -Before starting a fresh write test, clear out the test table using the Accumulo -shell. - - > deletetable -f stress_test - > createtable stress_test - -To run a writer: - - $ ./writer.sh - -The writer will begin logging into a "logs" subdirectory, using timestamped -filenames. You can stop the writer by killing it or using Control-C. - -To run a reader: - - $ ./reader.sh - -The reader logs like the writer. By default, the reader reads forever, but -it can be configured for a fixed number of scan iterations. - -### Group Execution - -To run writers across multiple machines, create a writers file and list each -host that will run a writer, one per line. - -Finally, to start and stop the writers: - - $ ./start-writers.sh - $ ./stop-writers.sh - -For readers, make a similar readers file, and use `start-readers.sh` and -`stop-readers.sh`. - -The start scripts copy out stress-env.sh to each of the writers and readers, -overwriting what is there, so you only need to edit the file on the host from -which you run those scripts. - -Note that in versions of Accumulo greater than 1.6.0, users should verify that -the cell sizes they supply meet the criteria set forth by the key constraints -on their tables. By default, tables are configured to reject keys that are -greater than 1MB. http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/reader.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/reader.sh b/test/system/stress/reader.sh deleted file mode 100755 index e282632..0000000 --- a/test/system/stress/reader.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -# Ref: http://stackoverflow.com/questions/59895/ -SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -# Stop: Resolve Script Directory -LOG_DIR=${DIR}/logs -mkdir -p "$LOG_DIR" - -# Source environment -. "${DIR}/stress-env.sh" - -ts=$(date +%Y%m%d%H%M%S) -host=$(hostname) -# We want USERPASS to word split -"${ACCUMULO_HOME}/bin/accumulo org.apache.accumulo.test.stress.random.Scan" "$INSTANCE" $USERPASS "$SCAN_SEED" "$CONTINUOUS_SCAN" "$SCAN_BATCH_SIZE" \ - > "$LOG_DIR/${ts}_${host}_reader.out" \ - 2> "$LOG_DIR/${ts}_${host}_reader.err" http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/readers ---------------------------------------------------------------------- diff --git a/test/system/stress/readers b/test/system/stress/readers deleted file mode 100644 index 80a4224..0000000 --- a/test/system/stress/readers +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -readerhost1 -readerhost2 http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/start-readers.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/start-readers.sh b/test/system/stress/start-readers.sh deleted file mode 100755 index 6f651f4..0000000 --- a/test/system/stress/start-readers.sh +++ /dev/null @@ -1,40 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -# Ref: http://stackoverflow.com/questions/59895/ -SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" -# Stop: Resolve Script Directory - -# Source environment -. "${DIR}/stress-env.sh" - -if [[ ! -f ${DIR}/readers ]]; then - echo readers file is missing - exit 1 -fi - -# Copy environment out -pscp -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}" - -pssh -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out 2>${DIR}/reader.err < /dev/null &" http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/start-writers.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/start-writers.sh b/test/system/stress/start-writers.sh deleted file mode 100755 index 474117c..0000000 --- a/test/system/stress/start-writers.sh +++ /dev/null @@ -1,40 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -# Ref: http://stackoverflow.com/questions/59895/ -SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -# Stop: Resolve Script Directory - -# Source environment -. "${DIR}/stress-env.sh" - -if [[ ! -f ${DIR}/writers ]]; then - echo writers file is missing - exit 1 -fi - -# Copy environment out -pscp -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}" - -pssh -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out 2>${DIR}/writer.err < /dev/null &" http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/stop-readers.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/stop-readers.sh b/test/system/stress/stop-readers.sh deleted file mode 100755 index 8bd4a6f..0000000 --- a/test/system/stress/stop-readers.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -# Ref: http://stackoverflow.com/questions/59895/ -SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -# Stop: Resolve Script Directory - -# Source environment -. "${DIR}/stress-env.sh" - -if [[ ! -f "${DIR}/readers" ]]; then - echo readers file is missing - exit 1 -fi -pssh -h "${DIR}/readers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/stop-writers.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/stop-writers.sh b/test/system/stress/stop-writers.sh deleted file mode 100755 index 5c9e8d7..0000000 --- a/test/system/stress/stop-writers.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -# Ref: http://stackoverflow.com/questions/59895/ -SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -# Stop: Resolve Script Directory - -# Source environment -. "${DIR}/stress-env.sh" - -if [[ ! -f "${DIR}/writers" ]]; then - echo writers file is missing - exit 1 -fi -pssh -h "${DIR}/writers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/stress-env.sh.example ---------------------------------------------------------------------- diff --git a/test/system/stress/stress-env.sh.example b/test/system/stress/stress-env.sh.example deleted file mode 100644 index 6efb24f..0000000 --- a/test/system/stress/stress-env.sh.example +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo} - -# Edit the credentials to match your system -USERPASS='-u root -p secret' -INSTANCE='-z localhost -i inst' - -### Read configuration ### - -# This is the seed for the range picking logic used by the scanner. -SCAN_SEED='--scan-seed 1337' - -# Controls the number of random tablets the scanner will read sequentially -#SCAN_ITERATIONS='--num-iterations 1024' - -# Alternatively, we can just continously scan -CONTINUOUS_SCAN='--continuous' - -# Controls whether or not the scan will be using an isolated scanner. Add this to the execution -#SCAN_ISOLATION='--isolate' - -# Sets the batch size for the scanner, use a lower number for large rows / cells -#SCAN_BATCH_SIZE='--scan-batch-size -1' - -### Write configuration ### - -# Edit these to change the range of each cell component. The size is in bytes. -ROW_RANGE='--min-row-size 128 --max-row-size 128' -CF_RANGE='--min-cf-size 128 --max-cf-size 128' -CQ_RANGE='--min-cq-size 128 --max-cq-size 128' -VALUE_RANGE='--min-value-size 1024 --max-value-size 2048' -ROW_WIDTH='--min-row-width 1 --max-row-width 10' - -# These are the seeds for the random number generates used to generate each cell component. -ROW_SEED='--row-seed 1' -CF_SEED='--cf-seed 2' -CQ_SEED='--cq-seed 3' -VALUE_SEED='--value-seed 4' -ROW_WIDTH_SEED='--row-width-seed 5' - -# This is the maximum number of cells to include in each mutation written out. -# A non-positive value implies no limit. -MAX_CELLS_PER_MUTATION='--max-cells-per-mutation -1' - -# This is the delay in milliseconds between writes. Use <= 0 for no delay. -WRITE_DELAY='--write-delay 0' http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/writer.sh ---------------------------------------------------------------------- diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh deleted file mode 100755 index 7107242..0000000 --- a/test/system/stress/writer.sh +++ /dev/null @@ -1,44 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -# Ref: http://stackoverflow.com/questions/59895/ -SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) - SOURCE=$(readlink "${SOURCE}") - [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd ) -# Stop: Resolve Script Directory -LOG_DIR=${DIR}/logs -mkdir -p "$LOG_DIR" - -# Source environment -. "${DIR}/stress-env.sh" - -ts=$(date +%Y%m%d%H%M%S) -host=$(hostname) - -# TBD - --clear-table option - -# We want $USERPASS to word split -"${ACCUMULO_HOME}/bin/accumulo org.apache.accumulo.test.stress.random.Write" "$INSTANCE" $USERPASS "$ROW_RANGE" "$CF_RANGE" "$CQ_RANGE" "$VALUE_RANGE" \ - "$ROW_SEED" "$CF_SEED" "$CQ_SEED" "$VALUE_SEED" \ - "$ROW_WIDTH" "$ROW_WIDTH_SEED" "$MAX_CELLS_PER_MUTATION" "$WRITE_DELAY" \ - > "$LOG_DIR/${ts}_${host}_writer.out" \ - 2> "$LOG_DIR/${ts}_${host}_writer.err" http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/stress/writers ---------------------------------------------------------------------- diff --git a/test/system/stress/writers b/test/system/stress/writers deleted file mode 100644 index 0a06590..0000000 --- a/test/system/stress/writers +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -writerhost1 -writerhost2 http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test1/README.md ---------------------------------------------------------------------- diff --git a/test/system/test1/README.md b/test/system/test1/README.md deleted file mode 100644 index 00767b7..0000000 --- a/test/system/test1/README.md +++ /dev/null @@ -1,46 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Command to run from command line - -Can run this test with pre-existing splits; use the following command to create the table with -100 pre-existing splits - -> `$ ../../../bin/accumulo 'org.apache.accumulo.test.TestIngest' --createTable \ --u root -p secret --splits 100 --rows 0` - -Could try running verify commands after stopping and restarting accumulo - -When write ahead log is implemented can try killing tablet server in middle of ingest - -Run 5 parallel ingesters and verify: - -> `$ . ingest_test.sh` -(wait) -`$ . verify_test.sh` -(wait) - -Overwrite previous ingest: -> `$ . ingest_test_2.sh` -(wait) -`$ . verify_test_2.sh` -(wait) - -Delete what was previously ingested: -> `$ . ingest_test_3.sh` -(wait) - http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test1/ingest_test.sh ---------------------------------------------------------------------- diff --git a/test/system/test1/ingest_test.sh b/test/system/test1/ingest_test.sh deleted file mode 100755 index 65bf2bd..0000000 --- a/test/system/test1/ingest_test.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 4000000 --cols 1 & http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test1/ingest_test_2.sh ---------------------------------------------------------------------- diff --git a/test/system/test1/ingest_test_2.sh b/test/system/test1/ingest_test_2.sh deleted file mode 100755 index c1c5f75..0000000 --- a/test/system/test1/ingest_test_2.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 4000000 --cols 1 & http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test1/ingest_test_3.sh ---------------------------------------------------------------------- diff --git a/test/system/test1/ingest_test_3.sh b/test/system/test1/ingest_test_3.sh deleted file mode 100755 index 650e3fe..0000000 --- a/test/system/test1/ingest_test_3.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 4000000 --cols 1 & http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test1/verify_test.sh ---------------------------------------------------------------------- diff --git a/test/system/test1/verify_test.sh b/test/system/test1/verify_test.sh deleted file mode 100755 index f91eb5f..0000000 --- a/test/system/test1/verify_test.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 1 --random 56 --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 1 --random 56 --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 1 --random 56 --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 1 --random 56 --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 1 --random 56 --rows 1000000 --start 4000000 --cols 1 & http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test1/verify_test_2.sh ---------------------------------------------------------------------- diff --git a/test/system/test1/verify_test_2.sh b/test/system/test1/verify_test_2.sh deleted file mode 100755 index 75e4119..0000000 --- a/test/system/test1/verify_test_2.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 2 --random 57 --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 2 --random 57 --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 2 --random 57 --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 2 --random 57 --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest -u root -p secret --size 50 --timestamp 2 --random 57 --rows 1000000 --start 4000000 --cols 1 & http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test2/README.md ---------------------------------------------------------------------- diff --git a/test/system/test2/README.md b/test/system/test2/README.md deleted file mode 100644 index e60539c..0000000 --- a/test/system/test2/README.md +++ /dev/null @@ -1,27 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Test Concurrent Read/Write -========================== - -Can run this test with pre-existing splits; use the following command to create the table with -100 pre-existing splits: - -> `$ ../../../bin/accumulo org.apache.accumulo.test.TestIngest --createTable \ --u root -p secret --splits 100 --rows 0 -$ . concurrent.sh` - http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test2/concurrent.sh ---------------------------------------------------------------------- diff --git a/test/system/test2/concurrent.sh b/test/system/test2/concurrent.sh deleted file mode 100755 index a759faf..0000000 --- a/test/system/test2/concurrent.sh +++ /dev/null @@ -1,99 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -USERPASS='-u root -p secret' -../../../bin/accumulo shell $USERPASS -e 'deletetable -f test_ingest' -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --rows 0 --createTable - -echo "ingesting first halves (0 to (500K - 1), 1M to (1.5M - 1), etc)" -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 4000000 --cols 1 & - -wait - -echo "ingesting second halves (500K to (1M - 1), 1.5M to (2M - 1), etc) and verifying first halves" - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 5000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 5000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 5000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 5000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 5000000 --start 4000000 --cols 1 & - -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 1500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 2500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 3500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 5000000 --start 4500000 --cols 1 & - -wait - -echo "verifying complete range" - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 4000000 --cols 1 & - -wait - -echo "ingesting first halves (0 to (500K - 1), 1M to (1.5M - 1), etc) w/ new timestamp AND verifying second half w/ old timestamp" - -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 4000000 --cols 1 & - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 500000 --start 500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 500000 --start 1500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 500000 --start 2500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 500000 --start 3500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 500000 --start 4500000 --cols 1 & - - -wait - -echo "ingesting second halves (500K to (1M - 1), 1.5M to (2M - 1), etc) and verifying first halves" - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 500000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 500000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 500000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 500000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 500000 --start 4000000 --cols 1 & - -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 1500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 2500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 3500000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 2 --size 50 --random 57 --rows 500000 --start 4500000 --cols 1 & - -wait - -echo "verifying complete range" - -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 1000000 --start 0 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 1000000 --start 1000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 1000000 --start 2000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 1000000 --start 3000000 --cols 1 & -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 2 --random 57 --rows 1000000 --start 4000000 --cols 1 & - - -wait http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test3/README.md ---------------------------------------------------------------------- diff --git a/test/system/test3/README.md b/test/system/test3/README.md deleted file mode 100644 index e63ae78..0000000 --- a/test/system/test3/README.md +++ /dev/null @@ -1,22 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Test Creating Big Non-Splittable Row -==================================== - -TBD - http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test3/bigrow.sh ---------------------------------------------------------------------- diff --git a/test/system/test3/bigrow.sh b/test/system/test3/bigrow.sh deleted file mode 100755 index e508146..0000000 --- a/test/system/test3/bigrow.sh +++ /dev/null @@ -1,27 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -USERPASS='-u root -p secret' -../../../bin/accumulo shell $USERPASS -e 'deletetable -f test_ingest' -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --rows 0 --createTable - -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 1 --start 5000000 --cols 2000000; -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 1000000 --start 0 --cols 1; -#../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 1 --start 5000000 --cols 2000000; -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 0 --cols 1; -../../../bin/accumulo org.apache.accumulo.test.TestIngest $USERPASS --timestamp 1 --size 50 --random 56 --rows 1000000 --start 7000000 --cols 1; -../../../bin/accumulo org.apache.accumulo.test.VerifyIngest $USERPASS --size 50 --timestamp 1 --random 56 --rows 1000000 --start 7000000 --cols 1; http://git-wip-us.apache.org/repos/asf/accumulo/blob/81f215c0/test/system/test4/README.md ---------------------------------------------------------------------- diff --git a/test/system/test4/README.md b/test/system/test4/README.md deleted file mode 100644 index 3be9182..0000000 --- a/test/system/test4/README.md +++ /dev/null @@ -1,26 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -Test Bulk Importing Data -======================== - -Can run this test with pre-existing splits... use the following command to create the table with -100 pre-existing splits - -> `../../../bin/accumulo org.apache.accumulo.test.TestIngest --createTable \ --u root -p secret --rows 0 --splits 100` -
