This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 102a9744bb8ed411d2828b974df93d178634be4e
Merge: d4249cfb18 11b1f48f03
Author: Daniel Roberts <ddani...@gmail.com>
AuthorDate: Sun Jul 28 21:21:40 2024 +0000
Merge branch 'main' into elasticity
assemble/bin/accumulo-cluster | 209 ++++++++++++++++++---
assemble/bin/accumulo-service | 140 +++++++++++---
.../core/conf/cluster/ClusterConfigParser.java | 23 ++-
.../core/conf/cluster/ClusterConfigParserTest.java | 26 ++-
.../cluster/cluster-with-optional-services.yaml | 9 +-
5 files changed, 329 insertions(+), 78 deletions(-)
diff --cc assemble/bin/accumulo-cluster
index b157f04c1e,5c4c50af52..e72eb52f04
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@@ -23,16 -23,20 +23,20 @@@ function print_usage
Usage: accumulo-cluster <command> (<argument> ...)
Commands:
- create-config Creates cluster config
- restart Restarts the Accumulo cluster
- start Starts Accumulo cluster
- stop Stops Accumulo cluster
- kill Kills Accumulo cluster
- start-non-tservers Starts all services except tservers
- start-tservers Starts all tservers on cluster
- stop-tservers Stops all tservers on cluster
- start-here Starts all services on this node
- stop-here Stops all services on this node
+ create-config Creates cluster config
+ restart Restarts the Accumulo cluster
+ start Starts Accumulo cluster
+ stop Stops Accumulo cluster
+ kill Kills Accumulo cluster
+ start-non-tservers Deprecated. Starts all services except tservers
- start-servers [--all|--tservers|--no-tservers|--sservers
[group]|--compactors [queue]]
- Starts various server types, can optionally
specify a queue or group
- stop-servers [--all|--tservers| --no-tservers|--sservers
[group]|--compactors [queue]]
- Starts various server types, can optionally
specify a queue or group
++ start-servers [--all|--tservers|--no-tservers|--sservers
[group]|--compactors [group]]
++ Starts various server types, can optionally
specify a group
++ stop-servers [--all|--tservers| --no-tservers|--sservers
[group]|--compactors [group]]
++ Starts various server types, can optionally
specify a group
+ start-tservers Deprecated. Starts all tservers on cluster
+ stop-tservers Deprecated. Stops all tservers on cluster
+ start-here Starts all services on this node
+ stop-here Stops all services on this node
EOF
}
@@@ -123,15 -111,19 +127,19 @@@ function parse_config
echo "INFO: ${NUM_TSERVERS} tservers will be started per host"
fi
- # shellcheck disable=SC2153
- if [[ -z $NUM_SSERVERS ]]; then
- echo "INFO: ${NUM_SSERVERS} sservers will be started per host"
- fi
-
- if [[ -z $NUM_COMPACTORS ]]; then
- echo "INFO: ${NUM_COMPACTORS} compactors will be started per host"
- fi
+ for group in $SSERVER_GROUPS; do
+ var_name="NUM_SSERVERS_${group}"
+ if [[ -n ${!var_name} ]]; then
+ echo "INFO: ${!var_name} scan servers will be started per host for
group ${group}"
+ fi
+ done
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
++ for group in $COMPACTOR_GROUPS; do
++ var_name="NUM_COMPACTORS_${group}"
+ if [[ -n ${!var_name} ]]; then
- echo "INFO: ${!var_name} compactors will be started per host for queue
${queue}"
++ echo "INFO: ${!var_name} compactors will be started per host for group
${group}"
+ fi
+ done
}
function control_service() {
@@@ -155,10 -138,10 +163,10 @@@
[[ $service == "compactor" ]] && last_instance_id=${NUM_COMPACTORS:-1}
for ((inst_id = 1; inst_id <= last_instance_id; inst_id++)); do
+ # Only increment the service name when more than one service is desired.
ACCUMULO_SERVICE_INSTANCE=""
- [[ $service == "tserver" && ${NUM_TSERVERS:-1} -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
- [[ $service == "compactor" ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
- [[ $service == "sserver" ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
- [[ last_instance_id -gt 1 ]] && ACCUMULO_SERVICE_INSTANCE="${inst_id}"
- [[ $service =~ ^compactor|sserver$ ]] &&
ACCUMULO_SERVICE_INSTANCE="${ACCUMULO_SERVICE_INSTANCE}_${5}"
++ [[ last_instance_id -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
++ [[ $service =~ ^compactor|sserver$ ]] &&
ACCUMULO_SERVICE_INSTANCE="${ACCUMULO_SERVICE_INSTANCE}_${inst_id}"
if [[ $host == localhost || $host == "$(hostname -s)" || $host ==
"$(hostname -f)" || "$(hostname -I)" =~ $host ]]; then
#
@@@ -185,20 -168,100 +193,104 @@@ function start_service()
control_service start "$@"
}
+ function start_compactors() {
+ echo -n "Starting compactor servers ..."
- queues=$COMPACTION_QUEUES
++ groups=$COMPACTOR_GROUPS
+ if [[ -n $1 ]]; then
- queues="$1"
- echo "Only starting servers for group: ${queues}"
++ groups="$1"
++ echo "Only starting servers for group: ${groups}"
+ fi
- for queue in $queues; do
- var_name="NUM_COMPACTORS_${queue}"
++ for group in $groups; do
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
++ Q="COMPACTOR_HOSTS_${group}"
+ if [[ -n ${!Q} ]]; then
+ for compactor in ${!Q}; do
- start_service "$compactor" compactor "-o" "compactor.queue=$queue"
++ start_service "$compactor" compactor "-o" "compactor.group=$group"
+ done
+ else
- echo "${queue} is not a valid queue ...exiting"
++ echo "${group} is not a valid compactor group ...exiting"
+ fi
+ done
+ }
+
+ function stop_compactors() {
+ echo "Stopping compactor servers ..."
- queues=$COMPACTION_QUEUES
++ groups=$COMPACTOR_GROUPS
+ if [[ -n $1 ]]; then
- queues="$1"
- echo "Only stopping servers for group: ${queues}"
++ groups="$1"
++ echo "Only stopping servers for group: ${groups}"
+ fi
- for queue in $queues; do
- var_name="NUM_COMPACTORS_${queue}"
++ for group in $groups; do
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
++ Q="COMPACTOR_HOSTS_${group}"
+ if [[ -n ${!Q} ]]; then
+ for compactor in ${!Q}; do
- stop_service "$compactor" compactor "-o" "compactor.queue=$queue"
++ stop_service "$compactor" compactor "-o" "compactor.group=$group"
+ done
+ else
- echo "${queue} is not a valid compaction queue ...exiting"
++ echo "${group} is not a valid compaction group ...exiting"
+ fi
+ done
+ }
+
+ function start_sservers() {
+ echo "Starting scan servers ..."
+ groups=$SSERVER_GROUPS
+ if [[ -n $1 ]]; then
+ groups="$1"
+ echo "Only starting servers for group: ${groups}"
+ fi
+ for group in $groups; do
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
+ G="SSERVER_HOSTS_${group}"
+ if [[ -n ${!G} ]]; then
+ for sserver in ${!G}; do
+ start_service "$sserver" sserver "-o" "sserver.group=$group"
+ done
+ else
+ echo "${group} is not a valid resource group ...exiting"
+ fi
+ done
+ }
+
+ function stop_sservers() {
+ echo "Stopping scan servers ..."
+ groups=$SSERVER_GROUPS
+ if [[ -n $1 ]]; then
+ groups="$1"
+ echo "Only stopping servers for group: ${groups}"
+ fi
+ for group in $groups; do
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
+ G="SSERVER_HOSTS_${group}"
+ if [[ -n ${!G} ]]; then
+ for sserver in ${!G}; do
+ stop_service "$sserver" sserver "-o" "sserver.group=$group"
+ done
+ else
- echo "${queue} is not a valid resource group ...exiting"
++ echo "${group} is not a valid resource group ...exiting"
+ fi
+ done
+ }
+
function start_tservers() {
- echo -n "Starting tablet servers ..."
- count=1
- for server in $TSERVER_HOSTS; do
- echo -n "."
- start_service "$server" tserver &
- if ((++count % 72 == 0)); then
- echo
- wait
- fi
+ echo "Starting tablet servers ..."
+ for group in $TSERVER_GROUPS; do
+ echo "Starting tablet servers for group $group"
+ count=1
+ G="TSERVER_HOSTS_${group}"
+ for tserver in ${!G}; do
+ echo -n "."
+ start_service "$tserver" tserver "-o" "tserver.group=$group"
+ if ((++count % 72 == 0)); then
+ echo
+ wait
+ fi
+ done
done
echo " done"
}
@@@ -223,18 -286,24 +315,22 @@@ function start_all()
done
for group in $SSERVER_GROUPS; do
+ echo "Starting scan servers for group $group"
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
G="SSERVER_HOSTS_${group}"
for sserver in ${!G}; do
start_service "$sserver" sserver "-o" "sserver.group=$group"
done
done
- for coordinator in $COORDINATOR_HOSTS; do
- start_service "$coordinator" compaction-coordinator
- done
-
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
+ for group in $COMPACTOR_GROUPS; do
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
+ echo "Starting compactors for group $group"
+ Q="COMPACTOR_HOSTS_${group}"
for compactor in ${!Q}; do
- start_service "$compactor" compactor "-o" "compactor.queue=$queue"
+ start_service "$compactor" compactor "-o" "compactor.group=$group"
done
done
@@@ -284,7 -350,8 +380,9 @@@ function start_here()
done
for group in $SSERVER_GROUPS; do
+ echo "Starting scan servers for group $group"
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
for host in $local_hosts; do
G="SSERVER_HOSTS_${group}"
for sserver in ${!G}; do
@@@ -295,13 -362,22 +393,15 @@@
done
done
- for host in $local_hosts; do
- for coordinator in $COORDINATOR_HOSTS; do
- if echo "$coordinator" | grep -q "^${host}\$"; then
- start_service "$coordinator" compaction-coordinator
- fi
- done
- done
-
- for queue in $COMPACTION_QUEUES; do
+ for group in $COMPACTOR_GROUPS; do
+ echo "Starting compactors for group $group"
for host in $local_hosts; do
- var_name="NUM_COMPACTORS_${queue}"
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
+ Q="COMPACTOR_HOSTS_${group}"
for compactor in ${!Q}; do
if echo "$compactor" | grep -q "^${host}\$"; then
- start_service "$compactor" compactor "-q" "$queue"
+ start_service "$compactor" compactor "-o" "compactor.group=$group"
fi
done
done
@@@ -363,26 -431,28 +463,30 @@@ function kill_all()
done
for group in $SSERVER_GROUPS; do
+ echo "Killing scan servers for group $group"
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
G="SSERVER_HOSTS_${group}"
for sserver in ${!G}; do
- kill_service "$sserver" sserver "-o" "sserver.group=$group"
+ kill_service "$sserver" sserver
done
done
- for host in $TSERVER_HOSTS; do
- kill_service "$host" tserver
- done
-
- for coordinator in $COORDINATOR_HOSTS; do
- kill_service "$coordinator" compaction-coordinator
+ for group in $TSERVER_GROUPS; do
+ echo "Killing tablet servers for group $group"
+ G="TSERVER_HOSTS_${group}"
+ for tserver in ${!G}; do
+ kill_service "$tserver" tserver
+ done
done
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
+ for group in $COMPACTOR_GROUPS; do
+ echo "Killing compactors for group $group"
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
+ Q="COMPACTOR_HOSTS_${group}"
for compactor in ${!Q}; do
- kill_service "$compactor" compactor "-o" "compactor.queue=$queue"
+ kill_service "$compactor" compactor
done
done
@@@ -421,18 -491,24 +525,22 @@@ function stop_all()
done
for group in $SSERVER_GROUPS; do
+ echo "Stopping scan servers for group $group"
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
G="SSERVER_HOSTS_${group}"
for sserver in ${!G}; do
- end_service $end_cmd "$sserver" sserver "-o" "sserver.group=$group"
+ end_service $end_cmd "$sserver" sserver
done
done
- for coordinator in $COORDINATOR_HOSTS; do
- end_service $end_cmd "$coordinator" compaction-coordinator
- done
-
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
+ for group in $COMPACTOR_GROUPS; do
+ echo "Stopping compactors for group $group"
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
+ Q="COMPACTOR_HOSTS_${group}"
for compactor in ${!Q}; do
- end_service $end_cmd "$compactor" compactor "-o"
"compactor.queue=$queue"
+ end_service $end_cmd "$compactor" compactor
done
done
@@@ -466,25 -539,23 +574,29 @@@ function stop_here()
for host in "${hosts_to_check[@]}"; do
for end_cmd in "stop" "kill"; do
- for svc in tserver gc manager monitor compaction-coordinator; do
+ for svc in gc manager monitor; do
end_service $end_cmd "$host" $svc
done
+ for group in $TSERVER_GROUPS; do
+ G="TSERVER_HOSTS_${group}"
+ for tserver in ${!G}; do
+ end_service $end_cmd "$tserver" tserver
+ done
+ done
for group in $SSERVER_GROUPS; do
+ var_name="NUM_SSERVERS_${group}"
+ [[ -n ${!var_name} ]] && NUM_SSERVERS=${!var_name}
G="SSERVER_HOSTS_${group}"
for sserver in ${!G}; do
- end_service $end_cmd "$sserver" sserver "-o" "sserver.group=$group"
+ end_service $end_cmd "$sserver" sserver
done
done
- for queue in $COMPACTION_QUEUES; do
- var_name="NUM_COMPACTORS_${queue}"
+ for group in $COMPACTOR_GROUPS; do
++ var_name="NUM_COMPACTORS_${group}"
+ [[ -n ${!var_name} ]] && NUM_COMPACTORS=${!var_name}
- Q="COMPACTOR_HOSTS_${queue}"
+ Q="COMPACTOR_HOSTS_${group}"
for compactor in ${!Q}; do
- end_service $end_cmd "$host" compactor "-o" "compactor.queue=$queue"
+ end_service $end_cmd "$host" compactor
done
done
done
diff --cc assemble/bin/accumulo-service
index 52787a7e23,9da187135d..0ff2813d78
--- a/assemble/bin/accumulo-service
+++ b/assemble/bin/accumulo-service
@@@ -97,11 -106,46 +105,46 @@@ function control_process()
fi
}
+ function find_processes() {
+ local service_type=$1
+ local file
+ for file in "$ACCUMULO_PID_DIR"/*; do
+ if file=$(expr "$file" : '^.*/accumulo-\('"$service_type"'.*\)[.]pid$');
then
+ RUNNING_PROCESSES+=("$file")
+ fi
+ done
+ }
+
+ function stop_service() {
+ local service_type=$1
+ local service_name=$2
+ local all_flag=$3
+ if $all_flag; then
+ find_processes "$service_type"
+ for process in "${RUNNING_PROCESSES[@]}"; do
+ local pid_file="${ACCUMULO_PID_DIR}/accumulo-${process}.pid"
+ control_process "TERM" "$process" "$pid_file"
+ done
+ else
+ echo "Stopping service process: $service_name"
- local pid_file="${ACCUMULO_PID_DIR}/accumulo-${service_name}.pid"
++ local pid_file="${ACCUMULO_PID_DIR}/accumulo-${service_name}-${group}.pid"
+ control_process "TERM" "$service_name" "$pid_file"
+ fi
+ }
+
function kill_service() {
- if [[ -f $pid_file ]]; then
- echo "Killing $service on $host"
- kill -s KILL "$(cat "$pid_file")" 2>/dev/null
- rm -f "${pid_file}" 2>/dev/null
+ local service_type=$1
+ local service_name=$2
+ local all_flag=$3
+ if $all_flag; then
+ find_processes "$service_type"
+ for process in "${RUNNING_PROCESSES[@]}"; do
+ local pid_file="${ACCUMULO_PID_DIR}/accumulo-${process}.pid"
+ control_process "KILL" "$process" "$pid_file"
+ done
+ else
- local pid_file="${ACCUMULO_PID_DIR}/accumulo-${service_name}.pid"
++ local pid_file="${ACCUMULO_PID_DIR}/accumulo-${service_name}-${group}.pid"
+ control_process "KILL" "$service_name" "$pid_file"
fi
}
@@@ -135,39 -191,51 +190,62 @@@ function main()
mkdir -p "$ACCUMULO_LOG_DIR" 2>/dev/null
mkdir -p "$ACCUMULO_PID_DIR" 2>/dev/null
- host="$(hostname)"
- if [[ -z $host ]]; then
- host=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut
-f1 -d'/')
+ HOST="$(hostname)"
+ if [[ -z $HOST ]]; then
+ HOST=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut
-f1 -d'/')
fi
- service="$1"
+ # Find the group parameter if any
+ GROUP_PATTERN="^(compactor.group|sserver.group|tserver.group)=(.*)$"
+ group="default"
+ for param in "$@"; do
+ if [[ $param =~ $GROUP_PATTERN ]]; then
+ group="${BASH_REMATCH[2]}"
+ fi
+ done
+
-
pid_file="${ACCUMULO_PID_DIR}/accumulo-${service}${ACCUMULO_SERVICE_INSTANCE}-${group}.pid"
- case "$service" in
+ local service_type="$1"
+ local command_name="$2"
+ shift 2
- local service_name="unknown"
++ local service_name=$service_type
+ local all_flag=false
+
+ # Check and see if accumulo-cluster is calling this script
+ if [[ -z $ACCUMULO_SERVICE_INSTANCE ]]; then
+ # The rest of the arguments are from a user
+ if [[ $1 == "--all" ]]; then
+ all_flag=true
+ else
+ # A named service has been specified
- service_name="$1"
++ if [[ $1 != "-o" ]]; then
++ service_name="$1"
++ fi
+ fi
+ # Use the special bash env var from accumulo-cluster
+ else
- service_name=${service_type}${ACCUMULO_SERVICE_INSTANCE}
++ service_name=${service_type}${ACCUMULO_SERVICE_INSTANCE}-${group}
+ fi
+
+ case "$service_type" in
- gc | manager | monitor | tserver | compaction-coordinator | compactor |
sserver)
+ gc | manager | monitor | tserver | compactor | sserver)
- if [[ -z $2 ]]; then
+ if [[ -z $command_name ]]; then
invalid_args "<command> cannot be empty"
fi
- case "$2" in
+ case "$command_name" in
start)
- start_service "${@:3}"
+ start_service "$service_type" "$service_name" "$@"
;;
stop)
- stop_service
+ stop_service "$service_type" "$service_name" $all_flag
;;
kill)
- kill_service
+ kill_service "$service_type" "$service_name" $all_flag
+ ;;
+ list)
+ list_processes "$service_type"
;;
*)
- invalid_args "'$2' is an invalid <command>"
+ invalid_args "'$command_name' is an invalid <command>"
;;
esac
;;
diff --cc
core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
index c4ebf8c7ec,a16b6186c5..967d58c146
---
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
@@@ -39,24 -39,13 +39,24 @@@ import edu.umd.cs.findbugs.annotations.
public class ClusterConfigParser {
private static final String PROPERTY_FORMAT = "%s=\"%s\"%n";
- private static final String[] SECTIONS = new String[] {"manager",
"monitor", "gc", "tserver"};
+ private static final String COMPACTOR_PREFIX = "compactor.";
- private static final String COMPACTORS_PER_HOST_KEY = "compactors_per_host";
++ private static final String COMPACTORS_PER_HOST_PREFIX =
"compactors_per_host.";
+ private static final String GC_KEY = "gc";
+ private static final String MANAGER_KEY = "manager";
+ private static final String MONITOR_KEY = "monitor";
+ private static final String SSERVER_PREFIX = "sserver.";
- private static final String SSERVERS_PER_HOST_KEY = "sservers_per_host";
++ private static final String SSERVERS_PER_HOST_PREFIX = "sservers_per_host.";
+ private static final String TSERVER_PREFIX = "tserver.";
+ private static final String TSERVERS_PER_HOST_KEY = "tservers_per_host";
+
+ private static final String[] UNGROUPED_SECTIONS =
+ new String[] {MANAGER_KEY, MONITOR_KEY, GC_KEY};
- private static final Set<String> VALID_CONFIG_KEYS = Set.of(MANAGER_KEY,
MONITOR_KEY, GC_KEY,
- SSERVERS_PER_HOST_KEY, TSERVERS_PER_HOST_KEY, COMPACTORS_PER_HOST_KEY);
+ private static final Set<String> VALID_CONFIG_KEYS =
- Set.of("manager", "monitor", "gc", "tserver", "tservers_per_host",
"compaction.coordinator");
++ Set.of(MANAGER_KEY, MONITOR_KEY, GC_KEY, TSERVERS_PER_HOST_KEY);
-- private static final Set<String> VALID_CONFIG_PREFIXES =
- Set.of(COMPACTOR_PREFIX, SSERVER_PREFIX, TSERVER_PREFIX);
- Set.of("compaction.compactor.", "sserver.", "compactors_per_host.",
"sservers_per_host.");
++ private static final Set<String> VALID_CONFIG_PREFIXES =
Set.of(COMPACTOR_PREFIX, SSERVER_PREFIX,
++ TSERVER_PREFIX, SSERVERS_PER_HOST_PREFIX, COMPACTORS_PER_HOST_PREFIX);
private static final Predicate<String> VALID_CONFIG_SECTIONS =
section -> VALID_CONFIG_KEYS.contains(section)
@@@ -124,16 -112,23 +124,18 @@@
}
}
- if (config.containsKey("compaction.coordinator")) {
- out.printf(PROPERTY_FORMAT, "COORDINATOR_HOSTS",
config.get("compaction.coordinator"));
- }
-
- String compactorPrefix = "compaction.compactor.";
- Set<String> compactorQueues =
- config.keySet().stream().filter(k -> k.startsWith(compactorPrefix))
- .map(k ->
k.substring(compactorPrefix.length())).collect(Collectors.toSet());
-
- if (!compactorQueues.isEmpty()) {
- out.printf(PROPERTY_FORMAT, "COMPACTION_QUEUES",
- compactorQueues.stream().collect(Collectors.joining(" ")));
- for (String queue : compactorQueues) {
- out.printf(PROPERTY_FORMAT, "COMPACTOR_HOSTS_" + queue,
- config.get("compaction.compactor." + queue));
- String numCompactors = config.getOrDefault("compactors_per_host." +
queue, "1");
- out.printf(PROPERTY_FORMAT, "NUM_COMPACTORS_" + queue, numCompactors);
+ List<String> compactorGroups =
+ config.keySet().stream().filter(k -> k.startsWith(COMPACTOR_PREFIX))
+ .map(k ->
k.substring(COMPACTOR_PREFIX.length())).sorted().collect(Collectors.toList());
+
+ if (!compactorGroups.isEmpty()) {
+ out.printf(PROPERTY_FORMAT, "COMPACTOR_GROUPS",
+ compactorGroups.stream().collect(Collectors.joining(" ")));
+ for (String group : compactorGroups) {
+ out.printf(PROPERTY_FORMAT, "COMPACTOR_HOSTS_" + group,
+ config.get(COMPACTOR_PREFIX + group));
++ String numCompactors = config.getOrDefault("compactors_per_host." +
group, "1");
++ out.printf(PROPERTY_FORMAT, "NUM_COMPACTORS_" + group, numCompactors);
}
}
@@@ -144,17 -140,9 +146,20 @@@
out.printf(PROPERTY_FORMAT, "SSERVER_GROUPS",
sserverGroups.stream().collect(Collectors.joining(" ")));
sserverGroups.forEach(ssg -> out.printf(PROPERTY_FORMAT,
"SSERVER_HOSTS_" + ssg,
- config.get(sserverPrefix + ssg)));
+ config.get(SSERVER_PREFIX + ssg)));
+ sserverGroups.forEach(ssg -> out.printf(PROPERTY_FORMAT,
"NUM_SSERVERS_" + ssg,
+ config.getOrDefault("sservers_per_host." + ssg, "1")));
++
+ }
+
+ List<String> tserverGroups = config.keySet().stream().filter(k ->
k.startsWith(TSERVER_PREFIX))
+ .map(k ->
k.substring(TSERVER_PREFIX.length())).sorted().collect(Collectors.toList());
+
+ if (!tserverGroups.isEmpty()) {
+ out.printf(PROPERTY_FORMAT, "TSERVER_GROUPS",
+ tserverGroups.stream().collect(Collectors.joining(" ")));
+ tserverGroups.forEach(tsg -> out.printf(PROPERTY_FORMAT,
"TSERVER_HOSTS_" + tsg,
+ config.get(TSERVER_PREFIX + tsg)));
}
String numTservers = config.getOrDefault("tservers_per_host", "1");
diff --cc
core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
index 189e48afc3,f0f27e629f..a50578914c
---
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
@@@ -84,7 -85,7 +84,8 @@@ public class ClusterConfigParserTest
Map<String,String> contents =
ClusterConfigParser.parseConfiguration(new
File(configFile.toURI()).getAbsolutePath());
- assertEquals(14, contents.size());
- assertEquals(16, contents.size());
++ assertEquals(17, contents.size());
++
assertTrue(contents.containsKey("manager"));
assertEquals("localhost1 localhost2", contents.get("manager"));
assertTrue(contents.containsKey("monitor"));
@@@ -103,19 -109,18 +104,25 @@@
assertEquals("hmvm1 hmvm2 hmvm3", contents.get("sserver.highmem"));
assertTrue(contents.containsKey("sserver.cheap"));
assertEquals("burstyvm1 burstyvm2", contents.get("sserver.cheap"));
+ assertFalse(contents.containsKey("tserver"));
+ assertTrue(contents.containsKey("tserver.default"));
+ assertEquals("localhost1 localhost2", contents.get("tserver.default"));
+ assertTrue(contents.containsKey("tserver.highmem"));
+ assertEquals("localhost3 localhost4", contents.get("tserver.highmem"));
+ assertTrue(contents.containsKey("tserver.cheap"));
+ assertEquals("localhost5 localhost6", contents.get("tserver.cheap"));
assertTrue(contents.containsKey("tservers_per_host"));
assertEquals("2", contents.get("tservers_per_host"));
- assertTrue(contents.containsKey("sservers_per_host"));
- assertEquals("2", contents.get("sservers_per_host"));
- assertTrue(contents.containsKey("compactors_per_host"));
- assertEquals("3", contents.get("compactors_per_host"));
+ assertTrue(contents.containsKey("sservers_per_host.default"));
+ assertEquals("1", contents.get("sservers_per_host.default"));
+ assertTrue(contents.containsKey("sservers_per_host.highmem"));
+ assertEquals("2", contents.get("sservers_per_host.highmem"));
+ assertTrue(contents.containsKey("sservers_per_host.cheap"));
+ assertEquals("3", contents.get("sservers_per_host.cheap"));
+ assertTrue(contents.containsKey("compactors_per_host.q1"));
+ assertEquals("3", contents.get("compactors_per_host.q1"));
+ assertTrue(contents.containsKey("compactors_per_host.q2"));
+ assertEquals("1", contents.get("compactors_per_host.q2"));
}
@Test
@@@ -171,11 -176,8 +178,9 @@@
expected.put("MANAGER_HOSTS", "localhost1 localhost2");
expected.put("MONITOR_HOSTS", "localhost1 localhost2");
expected.put("GC_HOSTS", "localhost");
- expected.put("TSERVER_HOSTS", "localhost1 localhost2 localhost3
localhost4");
+ expected.put("TSERVER_GROUPS", "default");
+ expected.put("TSERVER_HOSTS_default", "localhost1 localhost2 localhost3
localhost4");
expected.put("NUM_TSERVERS", "${NUM_TSERVERS:=1}");
- expected.put("NUM_SSERVERS", "${NUM_SSERVERS:=1}");
- expected.put("NUM_COMPACTORS", "${NUM_COMPACTORS:=1}");
expected.replaceAll((k, v) -> '"' + v + '"');
@@@ -223,20 -225,21 +228,23 @@@
expected.put("MANAGER_HOSTS", "localhost1 localhost2");
expected.put("MONITOR_HOSTS", "localhost1 localhost2");
expected.put("GC_HOSTS", "localhost");
- expected.put("TSERVER_HOSTS", "localhost1 localhost2 localhost3
localhost4");
- expected.put("COORDINATOR_HOSTS", "localhost1 localhost2");
- expected.put("COMPACTION_QUEUES", "q1 q2");
+ expected.put("COMPACTOR_GROUPS", "q1 q2");
expected.put("COMPACTOR_HOSTS_q1", "localhost1 localhost2");
expected.put("COMPACTOR_HOSTS_q2", "localhost3 localhost4");
- expected.put("SSERVER_GROUPS", "default highmem cheap");
+ expected.put("SSERVER_GROUPS", "cheap default highmem");
+ expected.put("SSERVER_HOSTS_cheap", "burstyvm1 burstyvm2");
expected.put("SSERVER_HOSTS_default", "localhost1 localhost2");
expected.put("SSERVER_HOSTS_highmem", "hmvm1 hmvm2 hmvm3");
- expected.put("SSERVER_HOSTS_cheap", "burstyvm1 burstyvm2");
+ expected.put("TSERVER_GROUPS", "cheap default highmem");
+ expected.put("TSERVER_HOSTS_cheap", "localhost5 localhost6");
+ expected.put("TSERVER_HOSTS_default", "localhost1 localhost2");
+ expected.put("TSERVER_HOSTS_highmem", "localhost3 localhost4");
expected.put("NUM_TSERVERS", "${NUM_TSERVERS:=2}");
- expected.put("NUM_SSERVERS", "${NUM_SSERVERS:=2}");
- expected.put("NUM_COMPACTORS", "${NUM_COMPACTORS:=3}");
+ expected.put("NUM_COMPACTORS_q1", "3");
+ expected.put("NUM_COMPACTORS_q2", "1");
+ expected.put("NUM_SSERVERS_default", "1");
+ expected.put("NUM_SSERVERS_highmem", "2");
+ expected.put("NUM_SSERVERS_cheap", "3");
expected.replaceAll((k, v) -> {
return '"' + v + '"';
diff --cc
core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
index deb7bba0b4,a0f3c0850c..06b8e6f81c
---
a/core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
+++
b/core/src/test/resources/org/apache/accumulo/core/conf/cluster/cluster-with-optional-services.yaml
@@@ -51,14 -46,23 +51,19 @@@ sserver
- burstyvm1
- burstyvm2
-compaction:
- coordinator:
+compactor:
+ q1:
- localhost1
- localhost2
- compactor:
- - q1:
- - localhost1
- - localhost2
- - q2:
- - localhost3
- - localhost4
+ q2:
+ - localhost3
+ - localhost4
tservers_per_host: 2
- sservers_per_host: 2
- compactors_per_host: 3
+ sservers_per_host:
+ - default : 1
+ - highmem : 2
+ - cheap : 3
+ compactors_per_host:
+ - q1 : 3
+ - q2 : 1
