http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/libexec/load-env.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/load-env.sh b/assemble/libexec/load-env.sh new file mode 100755 index 0000000..3d7d9a1 --- /dev/null +++ b/assemble/libexec/load-env.sh @@ -0,0 +1,157 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sources accumulo-env.sh and verifies environment variables + +function verify_env_dir() { + property=$1 + directory=$2 + if [[ -z "$directory" ]]; then + echo "$property is not set. Please make sure it's set globally or in conf/accumulo-env.sh." + exit 1 + fi + if [[ ! -d "$directory" ]]; then + echo "$property=$directory is not a valid directory. Please make sure it's set correctly globally or in conf/accumulo-env.sh." + exit 1 + fi +} + +# Resolve a program to its installation directory +locationByProgram() +{ + RESULT=$( which "$1" ) + if [[ "$?" != 0 && -z "${RESULT}" ]]; then + echo "Cannot find '$1' and '$2' is not set in $ACCUMULO_CONF_DIR/accumulo-env.sh" + exit 1 + fi + while [ -h "${RESULT}" ]; do # resolve $RESULT until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$RESULT" )" && pwd )" + RESULT="$(readlink "${RESULT}")" + [[ "${RESULT}" != /* ]] && RESULT="${DIR}/${RESULT}" # if $RESULT was a relative symlink, we need to resolve it relative to the path where the symlink file was located + done + # find the relative home directory, accounting for an extra bin directory + RESULT=$(dirname "$(dirname "${RESULT}")") + echo "Auto-set ${2} to '${RESULT}'. To suppress this message, set ${2} in conf/accumulo-env.sh" + eval "${2}=${RESULT}" +} + +# Resolve base directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do + libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" +done +libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +basedir=$( cd -P "${libexec}"/.. && pwd ) + +export ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$basedir/conf}" + +if [[ -z $ACCUMULO_CONF_DIR || ! -d $ACCUMULO_CONF_DIR ]]; then + echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists" + exit 1 +fi + +if [[ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh || ! -f $ACCUMULO_CONF_DIR/accumulo-site.xml ]]; then + echo "The configuration files 'accumulo-env.sh' & 'accumulo-site.xml' must exist in $ACCUMULO_CONF_DIR" + echo "Run 'accumulo create-config' to create them or copy them from $ACCUMULO_CONF_DIR/examples" + echo "Follow the instructions in INSTALL.md to edit them for your environment." + exit 1 +fi + +source "$ACCUMULO_CONF_DIR/accumulo-env.sh" + +## Variables that must be set + +: "${ACCUMULO_TSERVER_OPTS:?"variable is not set in accumulo-env.sh"}" +: "${ACCUMULO_MASTER_OPTS:?"variable is not set in accumulo-env.sh"}" +: "${ACCUMULO_MONITOR_OPTS:?"variable is not set in accumulo-env.sh"}" +: "${ACCUMULO_GC_OPTS:?"variable is not set in accumulo-env.sh"}" +: "${ACCUMULO_SHELL_OPTS:?"variable is not set in accumulo-env.sh"}" +: "${ACCUMULO_GENERAL_OPTS:?"variable is not set in accumulo-env.sh"}" +: "${ACCUMULO_OTHER_OPTS:?"variable is not set in accumulo-env.sh"}" + +### Variables that are derived + +# If not set in accumulo-env.sh, set env variables by program location. +test -z "${JAVA_HOME}" && locationByProgram java JAVA_HOME +test -z "${HADOOP_PREFIX}" && locationByProgram hadoop HADOOP_PREFIX +test -z "${ZOOKEEPER_HOME}" && locationByProgram zkCli.sh ZOOKEEPER_HOME + +export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_PREFIX/etc/hadoop}" +export ACCUMULO_HOME="${ACCUMULO_HOME:-$basedir}" +export ACCUMULO_BIN_DIR="${ACCUMULO_BIN_DIR:-$basedir/bin}" +export ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$basedir/conf}" +export ACCUMULO_LIB_DIR="${ACCUMULO_LIB_DIR:-$basedir/lib}" +export ACCUMULO_LIBEXEC_DIR="${ACCUMULO_LIBEXEC_DIR:-$basedir/libexec}" +export ACCUMULO_LOG_DIR="${ACCUMULO_LOG_DIR:-$basedir/logs}" +export ACCUMULO_PID_DIR="${ACCUMULO_PID_DIR:-$basedir/run}" +export ACCUMULO_OPT_DIR="${ACCUMULO_OPT_DIR:-$basedir/opt}" + +# Make directories that may not exist +mkdir -p "${ACCUMULO_LOG_DIR}" 2>/dev/null +mkdir -p "${ACCUMULO_PID_DIR}" 2>/dev/null + +# Verify all directories exist +verify_env_dir "JAVA_HOME" "${JAVA_HOME}" +verify_env_dir "HADOOP_PREFIX" "${HADOOP_PREFIX}" +verify_env_dir "HADOOP_CONF_DIR" "${HADOOP_CONF_DIR}" +verify_env_dir "ZOOKEEPER_HOME" "${ZOOKEEPER_HOME}" +verify_env_dir "ACCUMULO_HOME" "${ACCUMULO_HOME}" +verify_env_dir "ACCUMULO_BIN_DIR" "${ACCUMULO_BIN_DIR}" +verify_env_dir "ACCUMULO_CONF_DIR" "${ACCUMULO_CONF_DIR}" +verify_env_dir "ACCUMULO_LIB_DIR" "${ACCUMULO_LIB_DIR}" +verify_env_dir "ACCUMULO_LIBEXEC_DIR" "${ACCUMULO_LIBEXEC_DIR}" +verify_env_dir "ACCUMULO_LOG_DIR" "${ACCUMULO_LOG_DIR}" +verify_env_dir "ACCUMULO_PID_DIR" "${ACCUMULO_PID_DIR}" +verify_env_dir "ACCUMULO_OPT_DIR" "${ACCUMULO_OPT_DIR}" + +## Verify Zookeeper installation +ZOOKEEPER_VERSION=$(find -L "$ZOOKEEPER_HOME" -maxdepth 1 -name "zookeeper-[0-9]*.jar" | head -1) +if [ -z "$ZOOKEEPER_VERSION" ]; then + echo "A Zookeeper JAR was not found in $ZOOKEEPER_HOME." + echo "Please check ZOOKEEPER_HOME, either globally or in accumulo-env.sh." + exit 1 +fi +ZOOKEEPER_VERSION=$(basename "${ZOOKEEPER_VERSION##*-}" .jar) + +if [[ "$ZOOKEEPER_VERSION" < "3.4.0" ]]; then + echo "WARN : Using Zookeeper $ZOOKEEPER_VERSION. Use version 3.4.0 or greater. Older versions may not work reliably."; +fi + +## Variables that have a default +export ACCUMULO_KILL_CMD=${ACCUMULO_KILL_CMD:-'kill -9 %p'} +export ACCUMULO_MONITOR_BIND_ALL=${ACCUMULO_MONITOR_BIND_ALL:-"true"} +export ACCUMULO_JAAS_CONF=${ACCUMULO_JAAS_CONF:-${ACCUMULO_CONF_DIR}/jaas.conf} +export ACCUMULO_KRB5_CONF=${ACCUMULO_KRB5_CONF:-${ACCUMULO_CONF_DIR}/krb5.conf} +export ACCUMULO_NUM_OUT_FILES=${ACCUMULO_NUM_OUT_FILES:-5} +export ACCUMULO_WATCHER=${ACCUMULO_WATCHER:-"false"} +export ACCUMULO_NUM_TSERVERS=${ACCUMULO_NUM_TSERVERS:-1} +export ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"false"} +export ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + +# Validate that ACCUMULO_NUM_TSERVERS is a positive integer +if ! [[ $ACCUMULO_NUM_TSERVERS =~ ^[0-9]+$ ]]; then + echo "ACCUMULO_NUM_TSERVERS, when defined in accumulo-env.sh, should be a positive number, is '$ACCUMULO_NUM_TSERVERS'" + exit 1 +fi + +export HADOOP_HOME=$HADOOP_PREFIX +export HADOOP_HOME_WARN_SUPPRESS=true + +# See HADOOP-7154 and ACCUMULO-847 +export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/libexec/service.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/service.sh b/assemble/libexec/service.sh new file mode 100755 index 0000000..b34d9ba --- /dev/null +++ b/assemble/libexec/service.sh @@ -0,0 +1,235 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function print_usage { + cat <<EOF +Usage: service.sh <command> (<argument> ...) + +Commands: + start <host> <service> Starts <service> on local <host> + stop <host> <service> <signal> Stops <service> using <signal> on local <host> + +EOF + exit 1 +} + +function invalid_args { + echo -e "Invalid arguments: $1\n" + print_usage + exit 1 +} + +rotate_log () { + logfile=$1; + max_retained=$2; + if [[ ! $max_retained =~ ^[0-9]+$ ]] || [[ $max_retained -lt 1 ]] ; then + echo "ACCUMULO_NUM_OUT_FILES should be a positive number, but was '$max_retained'" + exit 1 + fi + + if [ -f "$logfile" ]; then # rotate logs + while [ $max_retained -gt 1 ]; do + prev=`expr $max_retained - 1` + [ -f "$logfile.$prev" ] && mv -f "$logfile.$prev" "$logfile.$max_retained" + max_retained=$prev + done + mv -f "$logfile" "$logfile.$max_retained"; + fi +} + +function start_service() { + + if [[ $# -ne 2 ]]; then + invalid_args "start command expects these arguments: <host> <service>" + fi + + host="$1" + service="$2" + + address=$host + loghost=$host + + # When the hostname provided is the alias/shortname, try to use the FQDN to make + # sure we send the right address to the Accumulo process. + if [[ "$host" = "$(hostname -s)" ]]; then + host="$(hostname -f)" + address="$host" + fi + + if [[ ${service} == "monitor" && ${ACCUMULO_MONITOR_BIND_ALL} == "true" ]]; then + address="0.0.0.0" + fi + + COMMAND="${ACCUMULO_BIN_DIR}/accumulo" + if [ "${ACCUMULO_WATCHER}" = "true" ]; then + COMMAND="${ACCUMULO_LIBEXEC_DIR}/accumulo-watcher.sh ${loghost}" + fi + + OUTFILE="${ACCUMULO_LOG_DIR}/${service}_${loghost}.out" + ERRFILE="${ACCUMULO_LOG_DIR}/${service}_${loghost}.err" + + # Rotate the .out and .err files + rotate_log "$OUTFILE" ${ACCUMULO_NUM_OUT_FILES} + rotate_log "$ERRFILE" ${ACCUMULO_NUM_OUT_FILES} + + # NUMA sanity check + if [[ $ACCUMULO_NUM_TSERVERS -eq 1 && -n $TSERVER_NUMA_OPTIONS ]]; then + echo "TSERVER_NUMA_OPTIONS declared when ACCUMULO_NUM_TSERVERS is 1, use ACCUMULO_NUMACTL_OPTIONS instead" + exit 1 + fi + if [[ $ACCUMULO_NUM_TSERVERS -gt 1 && -n $TSERVER_NUMA_OPTIONS && ${#TSERVER_NUMA_OPTIONS[*]} -ne $ACCUMULO_NUM_TSERVERS ]]; then + echo "TSERVER_NUMA_OPTIONS is declared, but not the same size as ACCUMULO_NUM_TSERVERS" + exit 1 + fi + + if [[ "$service" != "tserver" || $ACCUMULO_NUM_TSERVERS -eq 1 ]]; then + # Check the pid file to figure out if its already running. + PID_FILE="${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${service}.pid" + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + if kill -0 $PID 2>/dev/null; then + # Starting an already-started service shouldn't be an error per LSB + echo "$host : $service already running (${PID})" + exit 0 + fi + fi + echo "Starting $service on $host" + + ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} + ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + NUMA=`which numactl 2>/dev/null` + NUMACTL_EXISTS="$?" + if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" + else + export NUMA_CMD="" + fi + + # Fork the process, store the pid + nohup ${NUMA_CMD} "$COMMAND" "${service}" --address "${address}" >"$OUTFILE" 2>"$ERRFILE" < /dev/null & + echo $! > ${PID_FILE} + + else + + S="$service" + for (( t=1; t<=$ACCUMULO_NUM_TSERVERS; t++)); do + + service="$S-$t" + + # Check the pid file to figure out if its already running. + PID_FILE="${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${service}.pid" + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + if kill -0 $PID 2>/dev/null; then + # Starting an already-started service shouldn't be an error per LSB + echo "$host : $service already running (${PID})" + continue + fi + fi + echo "Starting $service on $host" + + ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + ACCUMULO_NUMACTL_OPTIONS=${TSERVER_NUMA_OPTIONS[$t]} + if [[ "$ACCUMULO_ENABLE_NUMACTL" == "true" ]]; then + NUMA=`which numactl 2>/dev/null` + NUMACTL_EXISTS=$? + if [[ ( ${NUMACTL_EXISTS} -eq 0 ) ]]; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" + else + export NUMA_CMD="" + fi + fi + + # We want the files to be consistently named with the log files + # server_identifier_hostname.{out,err}, e.g. tserver_2_fqdn.out + OUTFILE="${ACCUMULO_LOG_DIR}/${S}_${t}_${loghost}.out" + ERRFILE="${ACCUMULO_LOG_DIR}/${S}_${t}_${loghost}.err" + + # Rotate the .out and .err files + rotate_log "$OUTFILE" ${ACCUMULO_NUM_OUT_FILES} + rotate_log "$ERRFILE" ${ACCUMULO_NUM_OUT_FILES} + + # Fork the process, store the pid + nohup ${NUMA_CMD} "$COMMAND" "${service}" --address "${address}" >"$OUTFILE" 2>"$ERRFILE" < /dev/null & + echo $! > ${PID_FILE} + + done + fi + + # Check the max open files limit and selectively warn + MAX_FILES_OPEN=$(ulimit -n) + + if [[ -n $MAX_FILES_OPEN ]] ; then + MAX_FILES_RECOMMENDED=${MAX_FILES_RECOMMENDED:-32768} + if (( MAX_FILES_OPEN < MAX_FILES_RECOMMENDED )) + then + echo "WARN : Max open files on $host is $MAX_FILES_OPEN, recommend $MAX_FILES_RECOMMENDED" >&2 + fi + fi +} + +function stop_service() { + + if [[ $# -ne 3 ]]; then + invalid_args "stop command expects these arguments: <host> <service> <signal>" + fi + + host=$1 + service=$2 + signal=$3 + + for pid_file in ${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${service}*.pid; do + if [[ -f ${pid_file} ]]; then + echo "Stopping $service on $host"; + kill -s "$signal" `cat ${pid_file}` 2>/dev/null + rm -f ${pid_file} 2>/dev/null + fi + done +} + +function main() { + # Resolve libexec directory + SOURCE="${BASH_SOURCE[0]}" + while [[ -h "$SOURCE" ]]; do + libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" + done + libexec="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + + source "$libexec"/load-env.sh + + ACCUMULO_IDENT_STRING=${ACCUMULO_IDENT_STRING:-$USER} + + if [[ -z $1 ]]; then + invalid_args "<command> cannot be empty" + fi + + case "$1" in + start) + start_service ${*:2} + ;; + stop) + stop_service ${*:2} + ;; + *) + invalid_args "'$1' is an invalid <command>" + ;; + esac +} + +main "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/libexec/templates/accumulo-env.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/templates/accumulo-env.sh b/assemble/libexec/templates/accumulo-env.sh new file mode 100644 index 0000000..078ce6f --- /dev/null +++ b/assemble/libexec/templates/accumulo-env.sh @@ -0,0 +1,90 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################ +# Variables that must be set +############################ + +export ACCUMULO_TSERVER_OPTS="${tServerHigh_tServerLow} " +export ACCUMULO_MASTER_OPTS="${masterHigh_masterLow}" +export ACCUMULO_MONITOR_OPTS="${monitorHigh_monitorLow}" +export ACCUMULO_GC_OPTS="${gcHigh_gcLow}" +export ACCUMULO_SHELL_OPTS="${shellHigh_shellLow}" +export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true -XX:+CMSClassUnloadingEnabled" +export ACCUMULO_OTHER_OPTS="${otherHigh_otherLow}" + +####################################### +# Variables that are derived if not set +####################################### + +## If not set below, Accumulo will settings in environment or derive +## locations using by looking up java, hadoop, zkiCli.sh on PATH + +# export JAVA_HOME=/path/to/java +# export HADOOP_PREFIX=/path/to/hadoop +# export HADOOP_CONF_DIR=/path/to/hadoop/etc/conf +# export ZOOKEEPER_HOME=/path/to/zookeeper + +# If not set below, Accumulo will derive these locations by determining the root of your +# installation and using the default locations + +# export ACCUMULO_LOG_DIR=/path/to/accumulo/log + +#################################################### +# Variables that have a default. Uncomment to change +#################################################### + +## Specifies what do when the JVM runs out of heap memory +# export ACCUMULO_KILL_CMD='kill -9 %p' +## Should the monitor bind to all network interfaces -- default: true +# export ACCUMULO_MONITOR_BIND_ALL="true" +## Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins +# export ACCUMULO_JAAS_CONF=${ACCUMULO_CONF_DIR}/jaas.conf +## Location of krb5.conf file. Needed by Kerberos subsystems to find login servers +# export ACCUMULO_KRB5_CONF=${ACCUMULO_CONF_DIR}/krb5.conf +## The number of .out and .err files per process to retain +# export ACCUMULO_NUM_OUT_FILES=5 +## Should process be automatically restarted +# export ACCUMULO_WATCHER="false" +## Sets number of tablet servers to run on each node +# export ACCUMULO_NUM_TSERVERS=1 +## Enables NUMA control policy +# export ACCUMULO_ENABLE_NUMACTL="false" +## Sets NUMA control options +# export ACCUMULO_NUMACTL_OPTIONS="--interleave=all" + +############################################### +# Variables that are optional. Uncomment to set +############################################### + +## Example for configuring multiple tservers per host. Note that the ACCUMULO_NUMACTL_OPTIONS +## environment variable is used when ACCUMULO_NUM_TSERVERS is 1 to preserve backwards compatibility. +## If ACCUMULO_NUM_TSERVERS is greater than 2, then the TSERVER_NUMA_OPTIONS array is used if defined. +## If TSERVER_NUMA_OPTIONS is declared but not the correct size, then the service will not start. +# +# export ACCUMULO_NUM_TSERVERS=2 +# declare -a TSERVER_NUMA_OPTIONS +# TSERVER_NUMA_OPTIONS[1]="--cpunodebind 0" +# TSERVER_NUMA_OPTIONS[2]="--cpunodebind 1" + +## Optionally look for hadoop and accumulo native libraries for your +## platform in additional directories. (Use DYLD_LIBRARY_PATH on Mac OS X.) +## May not be necessary for Hadoop 2.x or using an RPM that installs to +## the correct system library directory. +# +# export LD_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${PLATFORM}:${LD_LIBRARY_PATH} + http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/libexec/templates/accumulo-site.xml ---------------------------------------------------------------------- diff --git a/assemble/libexec/templates/accumulo-site.xml b/assemble/libexec/templates/accumulo-site.xml new file mode 100644 index 0000000..ff490b1 --- /dev/null +++ b/assemble/libexec/templates/accumulo-site.xml @@ -0,0 +1,176 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless + you are simply testing at your workstation, you will most definitely need to change the three entries below. --> + + <property> + <name>instance.volumes</name> + <value></value> + <description>comma separated list of URIs for volumes. example: hdfs://localhost:9000/accumulo</description> + </property> + + <property> + <name>instance.zookeeper.host</name> + <value>localhost:2181</value> + <description>comma separated list of zookeeper servers</description> + </property> + + <property> + <name>instance.secret</name> + <value>DEFAULT</value> + <description>A secret unique to a given instance that all servers must know in order to communicate with one another. + Change it before initialization. To + change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd], + and then update this file. + </description> + </property> + + <property> + <name>tserver.memory.maps.max</name> + <value>${memMapMax}</value> + </property> + + <property> + <name>tserver.memory.maps.native.enabled</name> + <value>${nativeEnabled}</value> + </property> + + <property> + <name>tserver.cache.data.size</name> + <value>${cacheDataSize}</value> + </property> + + <property> + <name>tserver.cache.index.size</name> + <value>${cacheIndexSize}</value> + </property> + + <property> + <name>trace.token.property.password</name> + <!-- change this to the root user's password, and/or change the user below --> + <value>secret</value> + </property> + + <!-- Kerberos requirements --> + <property> + <name>instance.rpc.sasl.enabled</name> + <value>true</value> + </property> + + <property> + <name>general.kerberos.keytab</name> + <value>${keytab}</value> + </property> + + <property> + <name>general.kerberos.principal</name> + <value>${principal}</value> + </property> + + <property> + <name>trace.token.type</name> + <value>org.apache.accumulo.core.client.security.tokens.KerberosToken</value> + </property> + + <property> + <name>instance.security.authenticator</name> + <value>org.apache.accumulo.server.security.handler.KerberosAuthenticator</value> + </property> + + <property> + <name>instance.security.authorizor</name> + <value>org.apache.accumulo.server.security.handler.KerberosAuthorizor</value> + </property> + + <property> + <name>instance.security.permissionHandler</name> + <value>org.apache.accumulo.server.security.handler.KerberosPermissionHandler</value> + </property> + <!-- End Kerberos requirements --> + + <property> + <name>trace.user</name> + <value>${traceUser}</value> + </property> + + <property> + <name>tserver.sort.buffer.size</name> + <value>${sortBufferSize}</value> + </property> + + <property> + <name>tserver.walog.max.size</name> + <value>${waLogMaxSize}</value> + </property> +${mvnProjBaseDir} + <property> + <name>general.classpaths</name> + + <value> + <!-- Accumulo requirements --> + $ACCUMULO_HOME/lib/accumulo-server.jar, + $ACCUMULO_HOME/lib/accumulo-core.jar, + $ACCUMULO_HOME/lib/accumulo-start.jar, + $ACCUMULO_HOME/lib/accumulo-fate.jar, + $ACCUMULO_HOME/lib/accumulo-proxy.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + <!-- ZooKeeper requirements --> + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + <!-- Common Hadoop requirements --> + $HADOOP_CONF_DIR, + <!-- Hadoop 2 requirements --> + $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar, + <!-- End Hadoop 2 requirements --> + <!-- HDP 2.0 requirements --> + /usr/lib/hadoop/[^.].*.jar, + /usr/lib/hadoop/lib/[^.].*.jar, + /usr/lib/hadoop-hdfs/[^.].*.jar, + /usr/lib/hadoop-mapreduce/[^.].*.jar, + /usr/lib/hadoop-yarn/[^.].*.jar, + /usr/lib/hadoop-yarn/lib/jersey.*.jar, + <!-- End HDP 2.0 requirements --> + <!-- HDP 2.2 requirements --> + /usr/hdp/current/hadoop-client/[^.].*.jar, + /usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, + /usr/hdp/current/hadoop-hdfs-client/[^.].*.jar, + /usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar, + /usr/hdp/current/hadoop-yarn-client/[^.].*.jar, + /usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar, + /usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar + <!-- End HDP 2.2 requirements --> + <!-- IOP 4.1 requirements --> + /usr/iop/current/hadoop-client/[^.].*.jar, + /usr/iop/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, + /usr/iop/current/hadoop-hdfs-client/[^.].*.jar, + /usr/iop/current/hadoop-mapreduce-client/[^.].*.jar, + /usr/iop/current/hadoop-yarn-client/[^.].*.jar, + /usr/iop/current/hadoop-yarn-client/lib/jersey.*.jar, + /usr/iop/current/hive-client/lib/hive-accumulo-handler.jar + <!-- End IOP 4.1 requirements --> + </value> + <description>Classpaths that accumulo checks for updates and class files.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/libexec/tool.sh ---------------------------------------------------------------------- diff --git a/assemble/libexec/tool.sh b/assemble/libexec/tool.sh new file mode 100755 index 0000000..065ef6e --- /dev/null +++ b/assemble/libexec/tool.sh @@ -0,0 +1,92 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + libexec=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE="$libexec/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +libexec=$( cd -P "$( dirname "$SOURCE" )" && pwd ) +# Stop: Resolve Script Directory + +source "$libexec"/load-env.sh + +if [[ -z "$HADOOP_PREFIX" ]] ; then + echo "HADOOP_PREFIX is not set. Please make sure it's set globally or in conf/accumulo-env.sh" + exit 1 +fi +if [[ -z "$ZOOKEEPER_HOME" ]] ; then + echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" + exit 1 +fi + +ZOOKEEPER_CMD='ls -1 $ZOOKEEPER_HOME/zookeeper-[0-9]*[^csn].jar ' +if [[ $(eval $ZOOKEEPER_CMD | wc -l) -ne 1 ]] ; then + echo "Not exactly one zookeeper jar in $ZOOKEEPER_HOME" + exit 1 +fi +ZOOKEEPER_LIB=$(eval $ZOOKEEPER_CMD) + +LIB="$ACCUMULO_LIB_DIR" +CORE_LIB="$LIB/accumulo-core.jar" +FATE_LIB="$LIB/accumulo-fate.jar" +THRIFT_LIB="$LIB/libthrift.jar" +JCOMMANDER_LIB="$LIB/jcommander.jar" +COMMONS_VFS_LIB="$LIB/commons-vfs2.jar" +GUAVA_LIB="$LIB/guava.jar" +HTRACE_LIB="$LIB/htrace-core.jar" + +USERJARS=" " +for arg in "$@"; do + if [ "$arg" != "-libjars" -a -z "$TOOLJAR" ]; then + TOOLJAR="$arg" + shift + elif [ "$arg" != "-libjars" -a -z "$CLASSNAME" ]; then + CLASSNAME="$arg" + shift + elif [ -z "$USERJARS" ]; then + USERJARS=$(echo "$arg" | tr "," " ") + shift + elif [ "$arg" = "-libjars" ]; then + USERJARS="" + shift + else + break + fi +done + +LIB_JARS="$THRIFT_LIB,$CORE_LIB,$FATE_LIB,$ZOOKEEPER_LIB,$JCOMMANDER_LIB,$COMMONS_VFS_LIB,$GUAVA_LIB,$HTRACE_LIB" +H_JARS="$THRIFT_LIB:$CORE_LIB:$FATE_LIB:$ZOOKEEPER_LIB:$JCOMMANDER_LIB:$COMMONS_VFS_LIB:$GUAVA_LIB:$HTRACE_LIB" + +for jar in $USERJARS; do + LIB_JARS="$LIB_JARS,$jar" + H_JARS="$H_JARS:$jar" +done +export HADOOP_CLASSPATH="$H_JARS:$HADOOP_CLASSPATH" + +if [[ -z "$CLASSNAME" || -z "$TOOLJAR" ]]; then + echo "Usage: tool.sh path/to/myTool.jar my.tool.class.Name [-libjars my1.jar,my2.jar]" 1>&2 + exit 1 +fi + +#echo USERJARS=$USERJARS +#echo CLASSNAME=$CLASSNAME +#echo HADOOP_CLASSPATH=$HADOOP_CLASSPATH +#echo exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" $ARGS +exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/accumulo-watcher.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/accumulo-watcher.sh b/assemble/scripts/accumulo-watcher.sh deleted file mode 100755 index 9fb13a2..0000000 --- a/assemble/scripts/accumulo-watcher.sh +++ /dev/null @@ -1,133 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -LOGHOST=$1 -shift -process=$1 - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -# Stop: Resolve Script Directory - -. "${bin}"/config.sh - -CLEAN_EXIT="Clean Exit" -UNEXPECTED_EXCEPTION="Unexpected exception" -OOM_EXCEPTION="Out of memory exception" -ZKLOCK_LOST="ZKLock lost" -UNKNOWN_ERROR="Unknown error" - -ERRFILE=${ACCUMULO_LOG_DIR}/${process}_${LOGHOST}.err -OUTFILE=${ACCUMULO_LOG_DIR}/${process}_${LOGHOST}.out -DEBUGLOG=${ACCUMULO_LOG_DIR}/${process}_$(hostname).debug.log -export COMMAND="${bin}/accumulo \"\$@\"" - -logger -s "starting process $process at $(date)" -stopRunning="" -while [ -z "$stopRunning" ]; -do - eval $COMMAND 2> $ERRFILE - exit=$? - unset cause - if [ "$exit" -eq 0 ]; then - potentialStopRunning=$CLEAN_EXIT - elif [ "$exit" -eq 1 ]; then - potentialStopRunning=$UNEXPECTED_EXCEPTION - elif [ "$exit" -eq 130 ]; then - stopRunning="Control C detected, exiting" - elif [ "$exit" -eq 143 ]; then - stopRunning="Process terminated, exiting" - elif [ "$exit" -eq 137 ]; then - potentialStopRunning="Process killed, exiting" - fi - if [ -z "$stopRunning" ]; then - stopRunning=$potentialStopRunning; - - if [ $exit -eq 1 ]; then - source="exit code" - cause=$UNEXPECTED_EXCEPTION - elif tail -n50 $OUTFILE | grep "java.lang.OutOfMemoryError:" > /dev/null; then - source="logs" - cause=$OOM_EXCEPTION - elif [ "$process" = "tserver" ]; then - if tail -n50 $DEBUGLOG | grep "ERROR: Lost tablet server lock (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ "$process" = "master" ]; then - if tail -n50 $DEBUGLOG | grep "ERROR: Master lock in zookeeper lost (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ "$process" = "gc" ]; then - if tail -n50 $DEBUGLOG | grep "FATAL: GC lock in zookeeper lost (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ "$process" = "monitor" ]; then - if tail -n50 $DEBUGLOG | grep "ERROR: Monitor lock in zookeeper lost (reason =" > /dev/null ; then - source="logs" - cause=$ZKLOCK_LOST - fi - elif [ $exit -ne 0 ]; then - source="exit code" - cause=$UNKNOWN_ERROR - fi - case $cause in - #Unknown exit code - "$UNKNOWN_ERROR") - #window doesn't matter when retries = 0 - RETRIES=0 - ;; - - "$UNEXPECTED_EXCEPTION") - WINDOW=$UNEXPECTED_TIMESPAN - RETRIES=$UNEXPECTED_RETRIES - ;; - - "$OOM_EXCEPTION") - WINDOW=$OOM_TIMESPAN - RETRIES=$OOM_RETRIES - ;; - - "$ZLOCK_LOST") - WINDOW=$ZKLOCK_TIMESPAN - RETRIES=$ZKLOCK_RETRIES - ;; - esac - - if [ -n "$cause" ]; then - stopRunning="" - declare -i attempts - attempts="`jobs | grep "reason$cause" | wc -l`+1" - if [ "$RETRIES" -le $attempts ]; then - stopRunning="$process encountered $cause in $source with exit code $exit- quitting ($attempts/$RETRIES in $WINDOW seconds)" - # kill all sleeps now - for list in `jobs | cut -b 2-2`; do kill %$list; done - else - logger -s "$process encountered $cause in $source with exit code $exit- retrying ($attempts/$RETRIES in $WINDOW seconds)" - eval "(sleep $WINDOW ; echo "reason$cause" >> /dev/null) &" - fi - fi - fi -done -logger -s $stopRunning http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/bootstrap-config.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/bootstrap-config.sh b/assemble/scripts/bootstrap-config.sh deleted file mode 100755 index 44ae2ac..0000000 --- a/assemble/scripts/bootstrap-config.sh +++ /dev/null @@ -1,407 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function usage { - cat <<EOF -Usage: bootstrap_config.sh [-options] -where options include (long options not available on all platforms): - -d, --dir Alternate directory to setup config files - -s, --size Supported sizes: '1GB' '2GB' '3GB' '512MB' - -n, --native Configure to use native libraries - -j, --jvm Configure to use the jvm - -o, --overwrite Overwrite the default config directory - -v, --version Specify the Apache Hadoop version supported versions: '1' '2' - -k, --kerberos Configure for use with Kerberos - -h, --help Print this help message -EOF -} - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -# Stop: Resolve Script Directory - -# -# Resolve accumulo home for bootstrapping -# -ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd ) -TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates" -CONF_DIR="${ACCUMULO_HOME}/conf" -ACCUMULO_SITE=accumulo-site.xml -ACCUMULO_ENV=accumulo-env.sh - -SIZE= -TYPE= -HADOOP_VERSION= -OVERWRITE="0" -BASE_DIR= -KERBEROS= - -#Execute getopt -if [[ $(uname -s) == "Linux" ]]; then - args=$(getopt -o "b:d:s:njokv:h" -l "basedir:,dir:,size:,native,jvm,overwrite,kerberos,version:,help" -q -- "$@") -else # Darwin, BSD - args=$(getopt b:d:s:njokv:h $*) -fi - -#Bad arguments -if [[ $? != 0 ]]; then - usage 1>&2 - exit 1 -fi -eval set -- $args - -for i -do - case "$i" in - -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers - BASE_DIR=$2; shift - shift;; - -d|--dir) - CONF_DIR=$2; shift - shift;; - -s|--size) - SIZE=$2; shift - shift;; - -n|--native) - TYPE=native - shift;; - -j|--jvm) - TYPE=jvm - shift;; - -o|--overwrite) - OVERWRITE=1 - shift;; - -v|--version) - HADOOP_VERSION=$2; shift - shift;; - -k|--kerberos) - KERBEROS="true" - shift;; - -h|--help) - usage - exit 0 - shift;; - --) - shift - break;; - esac -done - -while [[ "${OVERWRITE}" = "0" ]]; do - if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then - echo "Warning your current config files in ${CONF_DIR} will be overwritten!" - echo - echo "How would you like to proceed?:" - select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do - if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then - echo -n "Please specifiy new conf directory: " - read CONF_DIR - elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then - OVERWRITE=1 - fi - break - done - else - OVERWRITE=1 - fi -done -echo "Copying configuration files to: ${CONF_DIR}" - -#Native 1GB -native_1GB_tServer="-Xmx128m -Xms128m" -_1GB_master="-Xmx128m -Xms128m" -_1GB_monitor="-Xmx64m -Xms64m" -_1GB_gc="-Xmx64m -Xms64m" -_1GB_other="-Xmx128m -Xms64m" -_1GB_shell="${_1GB_other}" - -_1GB_memoryMapMax="256M" -native_1GB_nativeEnabled="true" -_1GB_cacheDataSize="15M" -_1GB_cacheIndexSize="40M" -_1GB_sortBufferSize="50M" -_1GB_waLogMaxSize="256M" - -#Native 2GB -native_2GB_tServer="-Xmx256m -Xms256m" -_2GB_master="-Xmx256m -Xms256m" -_2GB_monitor="-Xmx128m -Xms64m" -_2GB_gc="-Xmx128m -Xms128m" -_2GB_other="-Xmx256m -Xms64m" -_2GB_shell="${_2GB_other}" - -_2GB_memoryMapMax="512M" -native_2GB_nativeEnabled="true" -_2GB_cacheDataSize="30M" -_2GB_cacheIndexSize="80M" -_2GB_sortBufferSize="50M" -_2GB_waLogMaxSize="512M" - -#Native 3GB -native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m" -_3GB_master="-Xmx1g -Xms1g" -_3GB_monitor="-Xmx1g -Xms256m" -_3GB_gc="-Xmx256m -Xms256m" -_3GB_other="-Xmx1g -Xms256m" -_3GB_shell="${_3GB_other}" - -_3GB_memoryMapMax="1G" -native_3GB_nativeEnabled="true" -_3GB_cacheDataSize="128M" -_3GB_cacheIndexSize="128M" -_3GB_sortBufferSize="200M" -_3GB_waLogMaxSize="1G" - -#Native 512MB -native_512MB_tServer="-Xmx48m -Xms48m" -_512MB_master="-Xmx128m -Xms128m" -_512MB_monitor="-Xmx64m -Xms64m" -_512MB_gc="-Xmx64m -Xms64m" -_512MB_other="-Xmx128m -Xms64m" -_512MB_shell="${_512MB_other}" - -_512MB_memoryMapMax="80M" -native_512MB_nativeEnabled="true" -_512MB_cacheDataSize="7M" -_512MB_cacheIndexSize="20M" -_512MB_sortBufferSize="50M" -_512MB_waLogMaxSize="100M" - -#JVM 1GB -jvm_1GB_tServer="-Xmx384m -Xms384m" - -jvm_1GB_nativeEnabled="false" - -#JVM 2GB -jvm_2GB_tServer="-Xmx768m -Xms768m" - -jvm_2GB_nativeEnabled="false" - -#JVM 3GB -jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G" - -jvm_3GB_nativeEnabled="false" - -#JVM 512MB -jvm_512MB_tServer="-Xmx128m -Xms128m" - -jvm_512MB_nativeEnabled="false" - - -if [[ -z "${SIZE}" ]]; then - echo "Choose the heap configuration:" - select DIRNAME in 1GB 2GB 3GB 512MB; do - echo "Using '${DIRNAME}' configuration" - SIZE=${DIRNAME} - break - done -elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB" && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then - echo "Invalid memory size" - echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'" - exit 1 -fi - -if [[ -z "${TYPE}" ]]; then - echo - echo "Choose the Accumulo memory-map type:" - select TYPENAME in Java Native; do - if [[ "${TYPENAME}" == "Native" ]]; then - TYPE="native" - echo "Don't forget to build the native libraries using the bin/build_native_library.sh script" - elif [[ "${TYPENAME}" == "Java" ]]; then - TYPE="jvm" - fi - echo "Using '${TYPE}' configuration" - echo - break - done -fi - -if [[ -z "${HADOOP_VERSION}" ]]; then - echo - echo "Choose the Apache Hadoop version:" - select HADOOP in 'Hadoop 2' 'HDP 2.0/2.1' 'HDP 2.2' 'IOP 4.1'; do - if [ "${HADOOP}" == "Hadoop 2" ]; then - HADOOP_VERSION="2" - elif [ "${HADOOP}" == "HDP 2.0/2.1" ]; then - HADOOP_VERSION="HDP2" - elif [ "${HADOOP}" == "HDP 2.2" ]; then - HADOOP_VERSION="HDP2.2" - elif [ "${HADOOP}" == "IOP 4.1" ]; then - HADOOP_VERSION="IOP4.1" - fi - echo "Using Hadoop version '${HADOOP_VERSION}' configuration" - echo - break - done -elif [[ "${HADOOP_VERSION}" != "2" && "${HADOOP_VERSION}" != "HDP2" && "${HADOOP_VERSION}" != "HDP2.2" ]]; then - echo "Invalid Hadoop version" - echo "Supported Hadoop versions: '2', 'HDP2', 'HDP2.2'" - exit 1 -fi - -TRACE_USER="root" - -if [[ ! -z "${KERBEROS}" ]]; then - echo - read -p "Enter server's Kerberos principal: " PRINCIPAL - read -p "Enter server's Kerberos keytab: " KEYTAB - TRACE_USER="${PRINCIPAL}" -fi - -for var in SIZE TYPE HADOOP_VERSION; do - if [[ -z ${!var} ]]; then - echo "Invalid $var configuration" - exit 1 - fi -done - -TSERVER="${TYPE}_${SIZE}_tServer" -MASTER="_${SIZE}_master" -MONITOR="_${SIZE}_monitor" -GC="_${SIZE}_gc" -SHELL="_${SIZE}_shell" -OTHER="_${SIZE}_other" - -MEMORY_MAP_MAX="_${SIZE}_memoryMapMax" -NATIVE="${TYPE}_${SIZE}_nativeEnabled" -CACHE_DATA_SIZE="_${SIZE}_cacheDataSize" -CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize" -SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize" -WAL_MAX_SIZE="_${SIZE}_waLogMaxSize" - -MAVEN_PROJ_BASEDIR="" - -if [[ ! -z "${BASE_DIR}" ]]; then - MAVEN_PROJ_BASEDIR="\n <property>\n <name>general.maven.project.basedir</name>\n <value>${BASE_DIR}</value>\n </property>\n" -fi - -#Configure accumulo-env.sh -mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/ -sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \ - -e "s/\${masterHigh_masterLow}/${!MASTER}/" \ - -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \ - -e "s/\${gcHigh_gcLow}/${!GC}/" \ - -e "s/\${shellHigh_shellLow}/${!SHELL}/" \ - -e "s/\${otherHigh_otherLow}/${!OTHER}/" \ - ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV - -#Configure accumulo-site.xml -sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \ - -e "s/\${nativeEnabled}/${!NATIVE}/" \ - -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \ - -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \ - -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \ - -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \ - -e "s=\${traceUser}=${TRACE_USER}=" \ - -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE - -# If we're not using kerberos, filter out the krb properties -if [[ -z "${KERBEROS}" ]]; then - sed -e 's/<!-- Kerberos requirements -->/<!-- Kerberos requirements --><!--/' \ - -e 's/<!-- End Kerberos requirements -->/--><!-- End Kerberos requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" -else - # Make the substitutions - sed -e "s!\${keytab}!${KEYTAB}!" \ - -e "s!\${principal}!${PRINCIPAL}!" \ - ${CONF_DIR}/${ACCUMULO_SITE} > temp - mv temp ${CONF_DIR}/${ACCUMULO_SITE} -fi - -# Configure hadoop version -if [[ "${HADOOP_VERSION}" == "2" ]]; then - sed -e 's/<!-- HDP 2.0 requirements -->/<!-- HDP 2.0 requirements --><!--/' \ - -e 's/<!-- End HDP 2.0 requirements -->/--><!-- End HDP 2.0 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- HDP 2.2 requirements -->/<!-- HDP 2.2 requirements --><!--/' \ - -e 's/<!-- End HDP 2.2 requirements -->/--><!-- End HDP 2.2 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- IOP 4.1 requirements -->/<!-- IOP 4.1 requirements --><!--/' \ - -e 's/<!-- End IOP 4.1 requirements -->/--><!-- End IOP 4.1 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" -elif [[ "${HADOOP_VERSION}" == "HDP2" ]]; then - sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements --><!--/' \ - -e 's/<!-- End Hadoop 2 requirements -->/--><!-- End Hadoop 2 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- HDP 2.2 requirements -->/<!-- HDP 2.2 requirements --><!--/' \ - -e 's/<!-- End HDP 2.2 requirements -->/--><!-- End HDP 2.2 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- IOP 4.1 requirements -->/<!-- IOP 4.1 requirements --><!--/' \ - -e 's/<!-- End IOP 4.1 requirements -->/--><!-- End IOP 4.1 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" -elif [[ "${HADOOP_VERSION}" == "HDP2.2" ]]; then - sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements --><!--/' \ - -e 's/<!-- End Hadoop 2 requirements -->/--><!-- End Hadoop 2 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- HDP 2.0 requirements -->/<!-- HDP 2.0 requirements --><!--/' \ - -e 's/<!-- End HDP 2.0 requirements -->/--><!-- End HDP 2.0 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- IOP 4.1 requirements -->/<!-- IOP 4.1 requirements --><!--/' \ - -e 's/<!-- End IOP 4.1 requirements -->/--><!-- End IOP 4.1 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" -elif [[ "${HADOOP_VERSION}" == "IOP4.1" ]]; then - sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements --><!--/' \ - -e 's/<!-- End Hadoop 2 requirements -->/--><!-- End Hadoop 2 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- HDP 2.0 requirements -->/<!-- HDP 2.0 requirements --><!--/' \ - -e 's/<!-- End HDP 2.0 requirements -->/--><!-- End HDP 2.0 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" - sed -e 's/<!-- HDP 2.2 requirements -->/<!-- HDP 2.2 requirements --><!--/' \ - -e 's/<!-- End HDP 2.2 requirements -->/--><!-- End HDP 2.2 requirements -->/' \ - "${CONF_DIR}/$ACCUMULO_SITE" > temp - mv temp "${CONF_DIR}/$ACCUMULO_SITE" -fi - -#Additional setup steps for native configuration. -if [[ ${TYPE} == native ]]; then - if [[ $(uname) == Linux ]]; then - if [[ -z $HADOOP_PREFIX ]]; then - echo "WARNING: HADOOP_PREFIX not set, cannot automatically configure LD_LIBRARY_PATH to include Hadoop native libraries" - else - NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null) - if [[ -z $NATIVE_LIB ]]; then - echo -e "WARNING: The Hadoop native libraries could not be found for your sytem in: $HADOOP_PREFIX" - else - sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp - mv temp "${CONF_DIR}/$ACCUMULO_ENV" - echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH" - fi - fi - fi - echo -e "Please remember to compile the Accumulo native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed." -fi -echo "Setup complete" http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/bootstrap-hdfs.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/bootstrap-hdfs.sh b/assemble/scripts/bootstrap-hdfs.sh deleted file mode 100755 index 1d2964a..0000000 --- a/assemble/scripts/bootstrap-hdfs.sh +++ /dev/null @@ -1,90 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -# Stop: Resolve Script Directory - -. "$bin"/config.sh - -# -# Find the system context directory in HDFS -# -SYSTEM_CONTEXT_HDFS_DIR=$(grep -A1 "general.vfs.classpaths" "$ACCUMULO_CONF_DIR/accumulo-site.xml" | tail -1 | perl -pe 's/\s+<value>//; s/<\/value>//; s/,.+$//; s|[^/]+$||; print $ARGV[1]') - -if [ -z "$SYSTEM_CONTEXT_HDFS_DIR" ] -then - echo "Your accumulo-site.xml file is not set up for the HDFS Classloader. Please add the following to your accumulo-site.xml file where ##CLASSPATH## is one of the following formats:" - echo "A single directory: hdfs://host:port/directory/" - echo "A single directory with a regex: hdfs://host:port/directory/.*.jar" - echo "Multiple directories: hdfs://host:port/directory/.*.jar,hdfs://host:port/directory2/" - echo "" - echo "<property>" - echo " <name>general.vfs.classpaths</name>" - echo " <value>##CLASSPATH##</value>" - echo " <description>location of the jars for the default (system) context</description>" - echo "</property>" - exit 1 -fi - -# -# Create the system context directy in HDFS if it does not exist -# -"$HADOOP_PREFIX/bin/hadoop" fs -ls "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null -if [[ $? != 0 ]]; then - "$HADOOP_PREFIX/bin/hadoop" fs -mkdir "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null - if [[ $? != 0 ]]; then - echo "Unable to create classpath directory at $SYSTEM_CONTEXT_HDFS_DIR" - exit 1 - fi -fi - -# -# Replicate to all tservers to avoid network contention on startup -# -TSERVERS=$ACCUMULO_CONF_DIR/tservers -NUM_TSERVERS=$(egrep -v '(^#|^\s*$)' "$TSERVERS" | wc -l) - -#let each datanode service around 50 clients -REP=$(( NUM_TSERVERS / 50 )) -(( REP < 3 )) && REP=3 - -# -# Copy all jars in lib to the system context directory -# -"$HADOOP_PREFIX/bin/hadoop" fs -moveFromLocal "$ACCUMULO_HOME"/lib/*.jar "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -setrep -R $REP "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null - -# -# We need some of the jars in lib, copy them back out and remove them from the system context dir -# -"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" "$ACCUMULO_HOME/lib/." > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" "$ACCUMULO_HOME/lib/." > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" "$ACCUMULO_HOME/lib/." > /dev/null -"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" > /dev/null -for f in $(grep -v '^#' "$ACCUMULO_CONF_DIR/tservers") -do - rsync -ra --delete "$ACCUMULO_HOME" $(dirname "$ACCUMULO_HOME") -done http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/build-native-library.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/build-native-library.sh b/assemble/scripts/build-native-library.sh deleted file mode 100755 index c2531ee..0000000 --- a/assemble/scripts/build-native-library.sh +++ /dev/null @@ -1,74 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink - bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -script=$( basename "$SOURCE" ) -# Stop: Resolve Script Directory - - -lib=${bin}/../lib -native_tarball=${lib}/accumulo-native.tar.gz -final_native_target="${lib}/native" - -if [[ ! -f $native_tarball ]]; then - echo "Could not find native code artifact: ${native_tarball}"; - exit 1 -fi - -# Make the destination for the native library -mkdir -p "${final_native_target}" || exit 1 - -# Make a directory for us to unpack the native source into -TMP_DIR=$(mktemp -d /tmp/accumulo-native.XXXX) || exit 1 - -# Unpack the tarball to our temp directory -tar xf "${native_tarball}" -C "${TMP_DIR}" - -if [[ $? != 0 ]]; then - echo "Failed to unpack native tarball to ${TMP_DIR}" - exit 1 -fi - -# Move to the first (only) directory in our unpacked tarball -native_dir=$(find "${TMP_DIR}" -maxdepth 1 -mindepth 1 -type d) - -cd "${native_dir}" || exit 1 - -# Make the native library -export USERFLAGS="$@" -make - -# Make sure it didn't fail -if [[ $? != 0 ]]; then - echo "Make failed!" - exit 1 -fi - -# "install" the artifact -cp libaccumulo.* "${final_native_target}" || exit 1 - -# Clean up our temp directory -rm -rf "${TMP_DIR}" - -echo "Successfully installed native library" http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/check-tservers ---------------------------------------------------------------------- diff --git a/assemble/scripts/check-tservers b/assemble/scripts/check-tservers deleted file mode 100755 index 7f9850e..0000000 --- a/assemble/scripts/check-tservers +++ /dev/null @@ -1,199 +0,0 @@ -#! /usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script will check the configuration and uniformity of all the nodes in a cluster. -# Checks -# each node is reachable via ssh -# login identity is the same -# the physical memory is the same -# the mounts are the same on each machine -# a set of writable locations (typically different disks) are in fact writable -# -# In order to check for writable partitions, you must configure the WRITABLE variable below. -# - -import subprocess -import time -import select -import os -import sys -import fcntl -import signal -if not sys.platform.startswith('linux'): - sys.stderr.write('This script only works on linux, sorry.\n') - sys.exit(1) - -TIMEOUT = 5 -WRITABLE = [] -#WRITABLE = ['/srv/hdfs1', '/srv/hdfs2', '/srv/hdfs3'] - -def ssh(tserver, *args): - 'execute a command on a remote tserver and return the Popen handle' - handle = subprocess.Popen( ('ssh', '-o', 'StrictHostKeyChecking=no', '-q', '-A', '-n', tserver) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - handle.tserver = tserver - handle.finished = False - handle.out = '' - return handle - -def wait(handles, seconds): - 'wait for lots of handles simultaneously, and kill anything that doesn\'t return in seconds time\n' - 'Note that stdout will be stored on the handle as the "out" field and "finished" will be set to True' - handles = handles[:] - stop = time.time() + seconds - for h in handles: - fcntl.fcntl(h.stdout, fcntl.F_SETFL, os.O_NONBLOCK) - while handles and time.time() < stop: - wait = min(0, stop - time.time()) - handleMap = dict( [(h.stdout, h) for h in handles] ) - rd, wr, err = select.select(handleMap.keys(), [], [], wait) - for r in rd: - handle = handleMap[r] - while 1: - more = handle.stdout.read(1024) - if more == '': - handles.remove(handle) - handle.poll() - handle.wait() - handle.finished = True - handle.out += more - if len(more) < 1024: - break - for handle in handles: - os.kill(handle.pid, signal.SIGKILL) - handle.poll() - -def runAll(tservers, *cmd): - 'Run the given command on all the tservers, returns Popen handles' - handles = [] - for tserver in tservers: - handles.append(ssh(tserver, *cmd)) - wait(handles, TIMEOUT) - return handles - -def checkIdentity(tservers): - 'Ensure the login identity is consistent across the tservers' - handles = runAll(tservers, 'id', '-u', '-n') - bad = set() - myIdentity = os.popen('id -u -n').read().strip() - for h in handles: - if not h.finished or h.returncode != 0: - print '#', 'cannot look at identity on', h.tserver - bad.add(h.tserver) - else: - identity = h.out.strip() - if identity != myIdentity: - print '#', h.tserver, 'inconsistent identity', identity - bad.add(h.tserver) - return bad - -def checkMemory(tservers): - 'Run free on all tservers and look for weird results' - handles = runAll(tservers, 'free') - bad = set() - mem = {} - swap = {} - for h in handles: - if not h.finished or h.returncode != 0: - print '#', 'cannot look at memory on', h.tserver - bad.add(h.tserver) - else: - if h.out.find('Swap:') < 0: - print '#',h.tserver,'has no swap' - bad.add(h.tserver) - continue - lines = h.out.split('\n') - for line in lines: - if line.startswith('Mem:'): - mem.setdefault(line.split()[1],set()).add(h.tserver) - if line.startswith('Swap:'): - swap.setdefault(line.split()[1],set()).add(h.tserver) - # order memory sizes by most common - mems = sorted([(len(v), k, v) for k, v in mem.items()], reverse=True) - mostCommon = float(mems[0][1]) - for _, size, tservers in mems[1:]: - fract = abs(mostCommon - float(size)) / mostCommon - if fract > 0.05: - print '#',', '.join(tservers), ': unusual memory size', size - bad.update(tservers) - swaps = sorted([(len(v), k, v) for k, v in swap.items()], reverse=True) - mostCommon = float(mems[0][1]) - for _, size, tservers in swaps[1:]: - fract = abs(mostCommon - float(size) / mostCommon) - if fract > 0.05: - print '#',', '.join(tservers), ': unusual swap size', size - bad.update(tservers) - return bad - -def checkWritable(tservers): - 'Touch all the directories that should be writable by this user return any nodes that fail' - if not WRITABLE: - print '# WRITABLE value not configured, not checking partitions' - return [] - handles = runAll(tservers, 'touch', *WRITABLE) - bad = set() - for h in handles: - if not h.finished or h.returncode != 0: - bad.add(h.tserver) - print '#', h.tserver, 'some drives are not writable' - return bad - -def checkMounts(tservers): - 'Check the file systems that are mounted and report any that are unusual' - handles = runAll(tservers, 'mount') - mounts = {} - finished = set() - bad = set() - for handle in handles: - if handle.finished and handle.returncode == 0: - for line in handle.out.split('\n'): - words = line.split() - if len(words) < 5: continue - if words[4] == 'nfs': continue - if words[0].find(':/') >= 0: continue - mount = words[2] - mounts.setdefault(mount, set()).add(handle.tserver) - finished.add(handle.tserver) - else: - bad.add(handle.tserver) - print '#', handle.tserver, 'did not finish' - for m in sorted(mounts.keys()): - diff = finished - mounts[m] - if diff: - bad.update(diff) - print '#', m, 'not mounted on', ', '.join(diff) - return bad - -def main(argv): - if len(argv) < 1: - sys.stderr.write('Usage: check_tservers tservers\n') - sys.exit(1) - sys.stdin.close() - tservers = set() - for tserver in open(argv[0]): - hashPos = tserver.find('#') - if hashPos >= 0: - tserver = tserver[:hashPos] - tserver = tserver.strip() - if not tserver: continue - tservers.add(tserver) - bad = set() - for test in checkIdentity, checkMemory, checkMounts, checkWritable: - bad.update(test(tservers - bad)) - for tserver in sorted(tservers - bad): - print tserver - -main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/config-server.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/config-server.sh b/assemble/scripts/config-server.sh deleted file mode 100755 index a9ae645..0000000 --- a/assemble/scripts/config-server.sh +++ /dev/null @@ -1,85 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Guarantees that Accumulo and its environment variables are set for start -# and stop scripts. Should always be run after config.sh. -# -# Parameters checked by script -# ACCUMULO_VERIFY_ONLY set to skip actions that would alter the local filesystem -# -# Values set by script that can be user provided. If not provided script attempts to infer. -# MONITOR Machine to run monitor daemon on. Used by start-here.sh script -# -# Iff ACCUMULO_VERIFY_ONLY is not set, this script will -# * Check for standalone mode (lack of masters and tservers files) -# - Do appropriate set up -# * Ensure the presense of local role files (masters, tservers, gc, tracers) -# -# Values always set by script. -# SSH Default ssh parameters used to start daemons -# - -unset MASTER1 -if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then - MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) -fi - -if [[ -z "${MONITOR}" ]] ; then - MONITOR=$MASTER1 - if [[ -f "$ACCUMULO_CONF_DIR/monitor" ]]; then - MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) - fi - if [[ -z "${MONITOR}" ]] ; then - echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." - exit 1 - fi -fi -if [[ ! -f "$ACCUMULO_CONF_DIR/tracers" && -z "${ACCUMULO_VERIFY_ONLY}" ]]; then - if [[ -z "${MASTER1}" ]] ; then - echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." - exit 1 - else - echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers" - fi - -fi - -if [[ ! -f "$ACCUMULO_CONF_DIR/gc" && -z "${ACCUMULO_VERIFY_ONLY}" ]]; then - if [[ -z "${MASTER1}" ]] ; then - echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." - exit 1 - else - echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc" - fi -fi - -SSH='ssh -qnf -o ConnectTimeout=2' - -# ACCUMULO-1985 provide a way to use the scripts and still bind to all network interfaces -export ACCUMULO_MONITOR_BIND_ALL=${ACCUMULO_MONITOR_BIND_ALL:-"false"} - -if [[ -z "${ACCUMULO_PID_DIR}" ]]; then - export ACCUMULO_PID_DIR="${ACCUMULO_HOME}/run" -fi -[[ -z ${ACCUMULO_VERIFY_ONLY} ]] && mkdir -p "${ACCUMULO_PID_DIR}" 2>/dev/null - -if [[ -z "${ACCUMULO_IDENT_STRING}" ]]; then - export ACCUMULO_IDENT_STRING="$USER" -fi - -# The number of .out and .err files to retain -export ACCUMULO_NUM_OUT_FILES=${ACCUMULO_NUM_OUT_FILES:-5} http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/config.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/config.sh b/assemble/scripts/config.sh deleted file mode 100755 index adc0e03..0000000 --- a/assemble/scripts/config.sh +++ /dev/null @@ -1,172 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Guarantees that Accumulo and its environment variables are set. -# -# Parameters checked by script -# ACCUMULO_VERIFY_ONLY set to skip actions that would alter the local filesystem -# -# Values set by script that can be user provided. If not provided script attempts to infer. -# ACCUMULO_CONF_DIR Location where accumulo-env.sh, accumulo-site.xml and friends will be read from -# ACCUMULO_HOME Home directory for Accumulo -# ACCUMULO_LOG_DIR Directory for Accumulo daemon logs -# ACCUMULO_VERSION Accumulo version name -# HADOOP_PREFIX Prefix to the home dir for hadoop. -# -# Iff ACCUMULO_VERIFY_ONLY is not set, this script will -# * Ensure the existence of ACCUMULO_LOG_DIR on the current host -# -# Values always set by script. -# MALLOC_ARENA_MAX To work around a memory management bug (see ACCUMULO-847) -# HADOOP_HOME Home dir for hadoop. TODO fix this. -# -# Values set by script if certain files exist -# ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins -# ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers -# -# NUMA related environment variables -# ACCUMULO_ENABLE_NUMACTL (Default: true) Use numactl if present on the system -# ACCUMULO_NUMACTL_OPTIONS (Default: --interleave=all) options to the numactl command - -if [ -z "${ACCUMULO_HOME}" ] ; then - # Start: Resolve Script Directory - SOURCE="${BASH_SOURCE[0]}" - while [[ -h $SOURCE ]]; do # resolve $SOURCE until the file is no longer a symlink - bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - script=$( basename "$SOURCE" ) - # Stop: Resolve Script Directory - - ACCUMULO_HOME=$( cd -P "${bin}"/.. && pwd ) - export ACCUMULO_HOME -fi - -if [[ ! -d ${ACCUMULO_HOME} ]]; then - echo "ACCUMULO_HOME=${ACCUMULO_HOME} is not a valid directory. Please make sure it exists" - exit 1 -fi - -export ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}" -if [[ -z $ACCUMULO_CONF_DIR || ! -d $ACCUMULO_CONF_DIR ]] -then - echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists" - exit 1 -fi - -if [[ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ]] ; then - . "$ACCUMULO_CONF_DIR/accumulo-env.sh" -elif [[ -z $ACCUMULO_TEST ]] ; then - # - # Attempt to bootstrap configuration and continue - # - echo - echo 'Accumulo is not properly configured.' - echo - echo 'Try running $ACCUMULO_HOME/bin/bootstrap_config.sh and then editing' - echo '$ACCUMULO_HOME/conf/accumulo-env.sh' - echo - exit 1 -fi - -if [[ -f $ACCUMULO_CONF_DIR/slaves ]]; then - echo "ERROR: A 'slaves' file was found in $ACCUMULO_CONF_DIR/" - echo "Accumulo now reads tablet server hosts from 'tservers' and requires that the 'slaves' file not be present to reduce confusion." - echo "Please rename the 'slaves' file to 'tservers' or remove it if both exist." - exit 1 -fi - -if [[ ! -f $ACCUMULO_CONF_DIR/tservers ]]; then - echo "ERROR: A 'tservers' file was not found at $ACCUMULO_CONF_DIR/tservers" - echo "Please make sure it exists and is configured with tablet server hosts." - exit 1 -fi - -[[ -z $ACCUMULO_LOG_DIR ]] && ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs -[[ -z $ACCUMULO_VERIFY_ONLY ]] && mkdir -p $ACCUMULO_LOG_DIR 2>/dev/null - -export ACCUMULO_LOG_DIR - -if [[ -z "$HADOOP_PREFIX" ]] -then - HADOOP_PREFIX="$(which hadoop)" - if [[ -z "$HADOOP_PREFIX" ]] - then - echo "You must set HADOOP_PREFIX" - exit 1 - fi - HADOOP_PREFIX=$(dirname "$HADOOP_PREFIX") - HADOOP_PREFIX=$(dirname "$HADOOP_PREFIX") -fi -if [[ ! -d "$HADOOP_PREFIX" ]] -then - echo "HADOOP_PREFIX, which has a value of $HADOOP_PREFIX, is not a directory." - exit 1 -fi -export HADOOP_PREFIX - -ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} -ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} -NUMA=`which numactl 2>/dev/null` -NUMACTL_EXISTS="$?" -if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then - export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" -else - export NUMA_CMD="" -fi - -NUM_TSERVERS=${NUM_TSERVERS:-1} - -# Validate that NUM_TSERVERS is a positive integer -if ! [[ $NUM_TSERVERS =~ ^[0-9]+$ ]]; then - echo "NUM_TSERVERS, when defined in accumulo-env.sh, should be a positive number, is '$NUM_TSERVERS'" - exit 1 -fi - -# NUMA sanity checks -if [[ $NUM_TSERVERS -eq 1 && -n $TSERVER_NUMA_OPTIONS ]]; then - echo "TSERVER_NUMA_OPTIONS declared when NUM_TSERVERS is 1, use ACCUMULO_NUMACTL_OPTIONS instead" - exit 1 -fi -if [[ $NUM_TSERVERS -gt 1 && -n $TSERVER_NUMA_OPTIONS && ${#TSERVER_NUMA_OPTIONS[*]} -ne $NUM_TSERVERS ]]; then - echo "TSERVER_NUMA_OPTIONS is declared, but not the same size as NUM_TSERVERS" - exit 1 -fi - -export HADOOP_HOME=$HADOOP_PREFIX -export HADOOP_HOME_WARN_SUPPRESS=true - -# See HADOOP-7154 and ACCUMULO-847 -export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1} - -# Check for jaas.conf configuration -if [[ -z ${ACCUMULO_JAAS_CONF} ]]; then - if [[ -f ${ACCUMULO_CONF_DIR}/jaas.conf ]]; then - export ACCUMULO_JAAS_CONF=${ACCUMULO_CONF_DIR}/jaas.conf - fi -fi - -# Check for krb5.conf configuration -if [[ -z ${ACCUMULO_KRB5_CONF} ]]; then - if [[ -f ${ACCUMULO_CONF_DIR}/krb5.conf ]]; then - export ACCUMULO_KRB5_CONF=${ACCUMULO_CONF_DIR}/krb5.conf - fi -fi - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/generate-monitor-certificate.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/generate-monitor-certificate.sh b/assemble/scripts/generate-monitor-certificate.sh deleted file mode 100755 index 5e5745f..0000000 --- a/assemble/scripts/generate-monitor-certificate.sh +++ /dev/null @@ -1,84 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -# Stop: Resolve Script Directory - -. "$bin"/config.sh - -ALIAS="default" -KEYPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) -STOREPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) -KEYSTOREPATH="$ACCUMULO_HOME/conf/keystore.jks" -TRUSTSTOREPATH="$ACCUMULO_HOME/conf/cacerts.jks" -CERTPATH="$ACCUMULO_HOME/conf/server.cer" - -if [[ -e "$KEYSTOREPATH" ]]; then - rm -i "$KEYSTOREPATH" - if [[ -e "$KEYSTOREPATH" ]]; then - echo "KeyStore already exists, exiting" - exit 1 - fi -fi - -if [[ -e "$TRUSTSTOREPATH" ]]; then - rm -i "$TRUSTSTOREPATH" - if [[ -e "$TRUSTSTOREPATH" ]]; then - echo "TrustStore already exists, exiting" - exit 2 - fi -fi - -if [[ -e "$CERTPATH" ]]; then - rm -i "$CERTPATH" - if [[ -e "$CERTPATH" ]]; then - echo "Certificate already exists, exiting" - exit 3 - fi -fi - -"${JAVA_HOME}/bin/keytool" -genkey -alias "$ALIAS" -keyalg RSA -keypass "$KEYPASS" -storepass "$KEYPASS" -keystore "$KEYSTOREPATH" -"${JAVA_HOME}/bin/keytool" -export -alias "$ALIAS" -storepass "$KEYPASS" -file "$CERTPATH" -keystore "$KEYSTOREPATH" -"${JAVA_HOME}/bin/keytool" -import -v -trustcacerts -alias "$ALIAS" -file "$CERTPATH" -keystore "$TRUSTSTOREPATH" -storepass "$STOREPASS" <<< "yes" - -echo -echo "keystore and truststore generated. now add the following to accumulo-site.xml:" -echo -echo " <property>" -echo " <name>monitor.ssl.keyStore</name>" -echo " <value>$KEYSTOREPATH</value>" -echo " </property>" -echo " <property>" -echo " <name>monitor.ssl.keyStorePassword</name>" -echo " <value>$KEYPASS</value>" -echo " </property>" -echo " <property>" -echo " <name>monitor.ssl.trustStore</name>" -echo " <value>$TRUSTSTOREPATH</value>" -echo " </property>" -echo " <property>" -echo " <name>monitor.ssl.trustStorePassword</name>" -echo " <value>$STOREPASS</value>" -echo " </property>" -echo http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/log-forwarder.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/log-forwarder.sh b/assemble/scripts/log-forwarder.sh deleted file mode 100755 index d625580..0000000 --- a/assemble/scripts/log-forwarder.sh +++ /dev/null @@ -1,50 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# This script executes a program that will forward some or all of the logs to a running instance of Chainsaw v2. -# To use this script, start Chainsaw on a host and create a new XMLSocketReceiver. This script -# accepts the following command line parameters -# -# host [required] - host running Chainsaw. Must be accessible via the network from this server -# port [required] - port that XMLSocketReceiver is listening on. -# filter [optional] - filter for log file names, * and ? are valid wildcards -# start [optional] - filter log messages beginning at this time (format is yyyyMMddHHmmss) -# end [optional] - filter log messages ending at this time (default is now, format is yyyyMMddHHmmss) -# level [optional] - filter log messages with this level and higher -# regex [optional] - filter log messages that match this regex (follows java.util.regex.Pattern syntax) -# -# -# Example: -# -# LogForwarder.sh -h 127.0.0.1 -p 4448 -f tserver* -s 2010010100001 -e 20100101235959 -l INFO -m .*scan.* -# - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) - SOURCE=$(readlink "$SOURCE") - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -script=$( basename "$SOURCE" ) -# Stop: Resolve Script Directory - -. "$bin"/config.sh - -"${JAVA_HOME}/bin/java" -cp "$ACCUMULO_HOME/lib" org.apache.accumulo.server.util.SendLogToChainsaw -d "$ACCUMULO_LOG_DIR" "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/158cf16d/assemble/scripts/start-all.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/start-all.sh b/assemble/scripts/start-all.sh deleted file mode 100755 index e2830aa..0000000 --- a/assemble/scripts/start-all.sh +++ /dev/null @@ -1,77 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -# Stop: Resolve Script Directory - -. "$bin"/config.sh -. "$bin"/config-server.sh -unset DISPLAY - -if [ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then - echo "${ACCUMULO_CONF_DIR}/accumulo-env.sh does not exist. Please make sure you configure Accumulo before you run anything" - echo "We provide examples you can copy in ${ACCUMULO_HOME}/conf/examples/ which are set up for your memory footprint" - exit 1 -fi - -if [ -z "$ZOOKEEPER_HOME" ] ; then - echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" - exit 1 -fi -if [ ! -d $ZOOKEEPER_HOME ]; then - echo "ZOOKEEPER_HOME is not a directory: $ZOOKEEPER_HOME" - echo "Please check the setting, either globally or in accumulo-env.sh." - exit 1 -fi - -ZOOKEEPER_VERSION=$(find -L $ZOOKEEPER_HOME -maxdepth 1 -name "zookeeper-[0-9]*.jar" | head -1) -if [ -z "$ZOOKEEPER_VERSION" ]; then - echo "A Zookeeper JAR was not found in $ZOOKEEPER_HOME." - echo "Please check ZOOKEEPER_HOME, either globally or in accumulo-env.sh." - exit 1 -fi -ZOOKEEPER_VERSION=$(basename "${ZOOKEEPER_VERSION##*-}" .jar) - -if [[ "$ZOOKEEPER_VERSION" < "3.4.0" ]]; then - echo "WARN : Using Zookeeper $ZOOKEEPER_VERSION. Use version 3.4.0 or greater. Older versions may not work reliably."; -fi - -${bin}/start-server.sh $MONITOR monitor - -if [ "$1" != "--notTservers" ]; then - ${bin}/tup.sh -fi - -${bin}/accumulo org.apache.accumulo.master.state.SetGoalState NORMAL -for master in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters"); do - ${bin}/start-server.sh $master master -done - -for gc in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/gc"); do - ${bin}/start-server.sh $gc gc -done - -for tracer in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"); do - ${bin}/start-server.sh $tracer tracer -done
