This is an automated email from the ASF dual-hosted git repository.
keith-turner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8e0064ad80 adds commands to import/export all config to/from yaml
(#6322)
8e0064ad80 is described below
commit 8e0064ad801bd1302ea21e9f09b794c5321a9069
Author: Keith Turner <[email protected]>
AuthorDate: Mon Apr 27 12:19:13 2026 -0700
adds commands to import/export all config to/from yaml (#6322)
These new command will make it easier to manage all accumulo
properties in a configuration management system like git. The
yaml produced has a very well defined sort order making it easy
to diff two exports from different times.
In follow on work can remove or deprecate some of the existing commands
that have subsets of similar functionality.
---
server/base/pom.xml | 4 +
.../accumulo/server/conf/store/PropStore.java | 9 +
.../server/conf/store/impl/ZooPropStore.java | 5 +
.../server/conf/util/ExportConfigCommand.java | 198 +++++
.../server/conf/util/ImportConfigCommand.java | 183 +++++
.../org/apache/accumulo/server/util/PropUtil.java | 2 +-
.../accumulo/test/conf/ImportExportConfigIT.java | 807 +++++++++++++++++++++
7 files changed, 1207 insertions(+), 1 deletion(-)
diff --git a/server/base/pom.xml b/server/base/pom.xml
index f9cf10de29..cc20e6f4f7 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -110,6 +110,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/store/PropStore.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/store/PropStore.java
index 9fe274e91e..74f89e806a 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/store/PropStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/store/PropStore.java
@@ -82,6 +82,15 @@ public interface PropStore {
void replaceAll(PropStoreKey propStoreKey, long version, Map<String,String>
props)
throws ConcurrentModificationException;
+ /**
+ * Unconditionally replaces all properties with the map provided. If a
property is not included in
+ * the new map, the property will not be set.
+ *
+ * @throws IllegalStateException if the values cannot be written or if an
underlying store
+ * exception occurs.
+ */
+ void replaceAll(@NonNull PropStoreKey propStoreKey, @NonNull
Map<String,String> props);
+
/**
* Delete the store node from the underlying store.
*
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
index 43e57f1d14..cc47d0b64a 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
@@ -212,6 +212,11 @@ public class ZooPropStore implements PropStore,
PropChangeListener {
mutateVersionedProps(propStoreKey, VersionedProperties::addOrUpdate,
props);
}
+ @Override
+ public void replaceAll(@NonNull PropStoreKey propStoreKey, @NonNull
Map<String,String> props) {
+ mutateVersionedProps(propStoreKey, VersionedProperties::replaceAll, props);
+ }
+
@Override
public void replaceAll(@NonNull PropStoreKey propStoreKey, long version,
@NonNull Map<String,String> props) {
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/util/ExportConfigCommand.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/util/ExportConfigCommand.java
new file mode 100644
index 0000000000..f28c4fdf27
--- /dev/null
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/util/ExportConfigCommand.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.conf.util;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.function.ToIntFunction;
+
+import org.apache.accumulo.core.cli.ServerOpts;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.ResourceGroupId;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.conf.store.NamespacePropKey;
+import org.apache.accumulo.server.conf.store.PropStore;
+import org.apache.accumulo.server.conf.store.PropStoreKey;
+import org.apache.accumulo.server.conf.store.ResourceGroupPropKey;
+import org.apache.accumulo.server.conf.store.SystemPropKey;
+import org.apache.accumulo.server.conf.store.TablePropKey;
+import org.apache.accumulo.server.util.ServerKeywordExecutable;
+import org.apache.accumulo.start.spi.CommandGroup;
+import org.apache.accumulo.start.spi.CommandGroups;
+import org.apache.accumulo.start.spi.KeywordExecutable;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+import com.beust.jcommander.JCommander;
+import com.google.auto.service.AutoService;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+@AutoService(KeywordExecutable.class)
+public class ExportConfigCommand extends
ServerKeywordExecutable<ExportConfigCommand.Opts> {
+
+ public ExportConfigCommand() {
+ super(new Opts());
+ }
+
+ @Override
+ public String keyword() {
+ return "export";
+ }
+
+ @Override
+ public CommandGroup commandGroup() {
+ return CommandGroups.CONFIG;
+ }
+
+ @Override
+ public String description() {
+ return "Exports all accumulo configuration to yaml with well defined sort
order to support diff.";
+ }
+
+ public static class Opts extends ServerOpts {
+
+ }
+
+ // The order of the enums is the order they will be sorted in yaml
+ public enum Scope {
+ SYSTEM, RESOURCE_GROUP, NAMESPACE, TABLE
+ }
+
+ record ScopedProperties(Scope scope, String name, Map<String,String> props) {
+
+ private static final String SCOPE_KEY = "scope";
+ private static final String NAME_KEY = "name";
+ private static final String PROPERTIES_KEY = "properties";
+ private static final Set<String> KEYS = Set.of(SCOPE_KEY, NAME_KEY,
PROPERTIES_KEY);
+
+ private static SortedMap<String,String> extractProps(Map<?,?> map) {
+ Map<?,?> tmp = (Map<?,?>) map.get(PROPERTIES_KEY);
+ var props = new TreeMap<String,String>();
+ tmp.forEach((k, v) -> {
+ Preconditions.checkArgument(
+ v instanceof String || v instanceof Integer || v instanceof
Boolean,
+ "Unsupported yaml type %s %s", v.getClass().getName(), v);
+ props.put((String) k, v.toString());
+ });
+ return props;
+ }
+
+ ScopedProperties(Map<?,?> map) {
+ this(Scope.valueOf((String) map.get(SCOPE_KEY)), (String)
map.get(NAME_KEY),
+ extractProps(map));
+ Preconditions.checkArgument(KEYS.equals(map.keySet()),
+ "Unexpected keys in yaml : " + map.keySet());
+ }
+
+ Map<?,?> toMap() {
+ // use a linked hash map to control the order of fields in the yaml
+ var map = new LinkedHashMap<>();
+ map.put(SCOPE_KEY, scope.name());
+ map.put(NAME_KEY, name);
+ // use a treemap so the properties are sorted in yaml
+ TreeMap<String,Object> translatedProps = new TreeMap<>();
+ props.forEach((k, v) -> {
+ // Use some more specific types when possible this will make the yaml
formatting nicer.
+ if (v.equalsIgnoreCase("true")) {
+ translatedProps.put(k, Boolean.TRUE);
+ } else if (v.equalsIgnoreCase("false")) {
+ translatedProps.put(k, Boolean.FALSE);
+ } else if (v.matches("-?[0-9]+")) {
+ translatedProps.put(k, Long.valueOf(v));
+ } else {
+ translatedProps.put(k, v);
+ }
+ });
+ map.put(PROPERTIES_KEY, translatedProps);
+ return map;
+ }
+
+ }
+
+ static PropStoreKey getKey(Scope scope, String id) {
+ return switch (scope) {
+ case SYSTEM -> SystemPropKey.of();
+ case RESOURCE_GROUP -> ResourceGroupPropKey.of(ResourceGroupId.of(id));
+ case NAMESPACE -> NamespacePropKey.of(NamespaceId.of(id));
+ case TABLE -> TablePropKey.of(TableId.of(id));
+ };
+ }
+
+ private static ScopedProperties getProperties(PropStore propStore, Scope
scope, String name) {
+ return getProperties(propStore, scope, name, name);
+ }
+
+ private static ScopedProperties getProperties(PropStore propStore, Scope
scope, String id,
+ String name) {
+ var vprops = propStore.get(getKey(scope, id));
+ return new ScopedProperties(scope, name, vprops.asMap());
+ }
+
+ private static List<ScopedProperties> getAllProperties(ServerContext
context) throws Exception {
+ List<ScopedProperties> allProps = new ArrayList<>();
+
+ var propStore = context.getPropStore();
+
+ allProps.add(getProperties(propStore, Scope.SYSTEM, ""));
+
+ for (var rgid : context.resourceGroupOperations().list()) {
+ allProps.add(getProperties(propStore, Scope.RESOURCE_GROUP,
rgid.canonical()));
+ }
+
+ context.getNamespaceIdToNameMap().forEach((nsid, namespaceName) -> {
+ allProps.add(getProperties(propStore, Scope.NAMESPACE, nsid.canonical(),
namespaceName));
+
+ context.getTableMapping(nsid).createIdToQualifiedNameMap(namespaceName)
+ .forEach((tableId, tableName) -> {
+ allProps.add(getProperties(propStore, Scope.TABLE,
tableId.canonical(), tableName));
+ });
+
+ });
+
+ return allProps;
+ }
+
+ @VisibleForTesting
+ public static String export(ServerContext context) throws Exception {
+ List<ScopedProperties> allProps = getAllProperties(context);
+
+ DumperOptions dopts = new DumperOptions();
+ dopts.setPrettyFlow(true);
+ var yaml = new Yaml(dopts);
+
+ ToIntFunction<ScopedProperties> scopeToInt = sp -> sp.scope().ordinal();
+ Comparator<ScopedProperties> comp =
+
Comparator.comparingInt(scopeToInt).thenComparing(ScopedProperties::name);
+
+ return
yaml.dumpAll(allProps.stream().sorted(comp).map(ScopedProperties::toMap).iterator());
+ }
+
+ @Override
+ public void execute(JCommander cl, Opts options) throws Exception {
+ System.out.println(export(getServerContext()));
+ }
+}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/util/ImportConfigCommand.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/util/ImportConfigCommand.java
new file mode 100644
index 0000000000..5c926cf761
--- /dev/null
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/util/ImportConfigCommand.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.conf.util;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.ServerOpts;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.ResourceGroupId;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.conf.store.NamespacePropKey;
+import org.apache.accumulo.server.conf.store.PropStoreKey;
+import org.apache.accumulo.server.conf.store.ResourceGroupPropKey;
+import org.apache.accumulo.server.conf.store.SystemPropKey;
+import org.apache.accumulo.server.conf.store.TablePropKey;
+import org.apache.accumulo.server.conf.util.ExportConfigCommand.Scope;
+import
org.apache.accumulo.server.conf.util.ExportConfigCommand.ScopedProperties;
+import org.apache.accumulo.server.util.PropUtil;
+import org.apache.accumulo.server.util.ServerKeywordExecutable;
+import org.apache.accumulo.start.spi.CommandGroup;
+import org.apache.accumulo.start.spi.CommandGroups;
+import org.apache.accumulo.start.spi.KeywordExecutable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+
+@AutoService(KeywordExecutable.class)
+public class ImportConfigCommand extends
ServerKeywordExecutable<ImportConfigCommand.Opts> {
+
+ private static final Logger log =
LoggerFactory.getLogger(ImportConfigCommand.class);
+
+ public ImportConfigCommand() {
+ super(new Opts());
+ }
+
+ @Override
+ public String keyword() {
+ return "import";
+ }
+
+ @Override
+ public CommandGroup commandGroup() {
+ return CommandGroups.CONFIG;
+ }
+
+ @Override
+ public String description() {
+ return "Imports accumulo properties from a yaml file.";
+ }
+
+ public static class Opts extends ServerOpts {
+ @Parameter(names = "--ignore-extra",
+ description = "Proceed when Accumulo has extra tables, resource
groups, or namespaces that are not in yaml")
+ public boolean ignoreExtra = false;
+ @Parameter(names = "--dry-run", description = "Only validates the yaml
file, does not import.")
+ public boolean dryRun = false;
+
+ }
+
+ static PropStoreKey getKey(Scope scope, String name, ServerContext context) {
+ try {
+ return switch (scope) {
+ case SYSTEM -> SystemPropKey.of();
+ case RESOURCE_GROUP ->
ResourceGroupPropKey.of(ResourceGroupId.of(name));
+ case NAMESPACE -> NamespacePropKey.of(context.getNamespaceId(name));
+ case TABLE -> TablePropKey.of(context.getTableId(name));
+ };
+ } catch (NamespaceNotFoundException | TableNotFoundException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ record ScopeName(Scope scope, String name) {
+ }
+
+ private static Set<ScopeName> getAllScopeNames(ServerContext context) {
+ Set<ScopeName> all = new HashSet<>();
+
+ all.add(new ScopeName(Scope.SYSTEM, ""));
+
+ for (var rgid : context.resourceGroupOperations().list()) {
+ all.add(new ScopeName(Scope.RESOURCE_GROUP, rgid.canonical()));
+ }
+
+ context.getNamespaceIdToNameMap().forEach((nsid, namespaceName) -> {
+ all.add(new ScopeName(Scope.NAMESPACE, namespaceName));
+
+ context.getTableMapping(nsid).createIdToQualifiedNameMap(namespaceName)
+ .forEach((tableId, tableName) -> {
+ all.add(new ScopeName(Scope.TABLE, tableName));
+ });
+
+ });
+
+ return all;
+ }
+
+ private static void validate(ServerContext serverContext,
List<ScopedProperties> allProps,
+ boolean ignoreExtra) {
+ var scopeNamesInYaml = new HashSet<ScopeName>();
+ allProps.forEach(sp -> scopeNamesInYaml.add(new ScopeName(sp.scope(),
sp.name())));
+ var scopeNamesInAccumulo = getAllScopeNames(serverContext);
+
+ if (!scopeNamesInYaml.equals(scopeNamesInAccumulo)) {
+ boolean fail = false;
+ for (var scopeName : Sets.difference(scopeNamesInYaml,
scopeNamesInAccumulo)) {
+ log.error("{}:{} is only in yaml and not present in Accumulo",
scopeName.scope(),
+ scopeName.name());
+ fail = true;
+ }
+ if (!ignoreExtra) {
+ for (var scopeName : Sets.difference(scopeNamesInAccumulo,
scopeNamesInYaml)) {
+ log.error("{}:{} is only in Accumulo and not present in yaml",
scopeName.scope(),
+ scopeName.name());
+ fail = true;
+ }
+ }
+ if (fail) {
+ throw new IllegalArgumentException(
+ "Yaml and Accumulo do not have the same tables,namespaces, and/or
resource groups");
+ }
+ }
+
+ // validate all scope+name before attempting to update any scope+name
+ for (var scopedProps : allProps) {
+ var propStoreKey = getKey(scopedProps.scope(), scopedProps.name(),
serverContext);
+ PropUtil.validateProperties(serverContext, propStoreKey,
scopedProps.props());
+ }
+ }
+
+ @VisibleForTesting
+ public static void load(ServerContext serverContext, InputStream in, Opts
options) {
+ Yaml yaml = new Yaml();
+ List<ScopedProperties> allProps = new ArrayList<>();
+ for (var obj : yaml.loadAll(in)) {
+ allProps.add(new ScopedProperties((Map<?,?>) obj));
+ }
+
+ validate(serverContext, allProps, options.ignoreExtra);
+
+ if (!options.dryRun) {
+ var propStore = serverContext.getPropStore();
+
+ for (var sp : allProps) {
+ var propStoreKey = getKey(sp.scope(), sp.name(), serverContext);
+ propStore.replaceAll(propStoreKey, sp.props());
+ }
+ }
+ }
+
+ @Override
+ public void execute(JCommander cl, Opts options) throws Exception {
+ load(getServerContext(), System.in, options);
+ }
+}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/PropUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/PropUtil.java
index e8c6ce3d7f..7d8bd6f3a8 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/PropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/PropUtil.java
@@ -59,7 +59,7 @@ public final class PropUtil {
context.getPropStore().replaceAll(propStoreKey, version, properties);
}
- protected static void validateProperties(final ServerContext context,
+ public static void validateProperties(final ServerContext context,
final PropStoreKey propStoreKey, final Map<String,String> properties)
throws IllegalArgumentException {
for (Map.Entry<String,String> prop : properties.entrySet()) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/conf/ImportExportConfigIT.java
b/test/src/main/java/org/apache/accumulo/test/conf/ImportExportConfigIT.java
new file mode 100644
index 0000000000..13a0966ca7
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/conf/ImportExportConfigIT.java
@@ -0,0 +1,807 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.conf;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ResourceGroupId;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.conf.util.ExportConfigCommand;
+import org.apache.accumulo.server.conf.util.ImportConfigCommand;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.jupiter.api.Test;
+
+public class ImportExportConfigIT extends AccumuloClusterHarness {
+ private static final String YAML1 =
+ """
+ scope: SYSTEM
+ name: ''
+ properties:
+ compaction.service.maintenance.planner:
org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner
+ compaction.service.maintenance.planner.opts.groups: |2
+ [{"group":"prod_maintenance_compactors"}]
+ compaction.service.prod.planner:
org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner
+ compaction.service.prod.planner.opts.groups: |2
+ [{"group":"prod_small_compactors",
"maxSize":"256M"},{"group":"prod_large_compactors"}]
+ compaction.service.test.planner:
org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner
+ compaction.service.test.planner.opts.groups: |2
+ [{"group":"test_small_compactors",
"maxSize":"256M"},{"group":"test_large_compactors"}]
+ ---
+ scope: RESOURCE_GROUP
+ name: default
+ properties: {
+ }
+ ---
+ scope: RESOURCE_GROUP
+ name: prod
+ properties:
+ general.special.chars:
"!@#$%^&*()_\\n+-=<,>.?/\\t:;\\"''{[}]\\\\~`|"
+ sserver.cache.data.size: 16G
+ sserver.cache.index.size: 2G
+ tserver.cache.data.size: 32G
+ tserver.cache.index.size: 4G
+ ---
+ scope: RESOURCE_GROUP
+ name: prod_large_compactors
+ properties: {
+ compactor.failure.termination.threshold: 5,
+ compactor.wait.time.job.max: 3m
+ }
+ ---
+ scope: RESOURCE_GROUP
+ name: prod_maintenance_compactors
+ properties: {
+ }
+ ---
+ scope: RESOURCE_GROUP
+ name: prod_small_compactors
+ properties: {
+ compactor.failure.termination.threshold: 10
+ }
+ ---
+ scope: RESOURCE_GROUP
+ name: test
+ properties: {
+ tserver.cache.data.size: 2G,
+ tserver.cache.index.size: 512M
+ }
+ ---
+ scope: RESOURCE_GROUP
+ name: test_large_compactors
+ properties: {
+ compactor.wait.time.job.max: 20s,
+ compactor.wait.time.job.min: 10ms
+ }
+ ---
+ scope: RESOURCE_GROUP
+ name: test_small_compactors
+ properties: {
+ compactor.wait.time.job.max: 10s,
+ compactor.wait.time.job.min: 10ms
+ }
+ ---
+ scope: NAMESPACE
+ name: ''
+ properties: {
+ }
+ ---
+ scope: NAMESPACE
+ name: accumulo
+ properties: {
+ }
+ ---
+ scope: NAMESPACE
+ name: production
+ properties: {
+ table.compaction.dispatcher.opts.service: prod,
+ table.compaction.dispatcher.opts.service.user: maintenance,
+ table.custom.assignment.group: prod
+ }
+ ---
+ scope: NAMESPACE
+ name: testing
+ properties: {
+ table.compaction.dispatcher.opts.service: test,
+ table.custom.assignment.group: test
+ }
+ ---
+ scope: TABLE
+ name: accumulo.fate
+ properties: {
+ table.cache.block.enable: true,
+ table.cache.index.enable: true,
+ table.compaction.major.ratio: 1,
+ table.durability: sync,
+ table.failures.ignore: false,
+ table.file.compress.blocksize: 32K,
+ table.file.replication: 5,
+ table.group.txAdmin: txadmin,
+ table.groups.enabled: txAdmin,
+ table.iterator.majc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 1,
+ table.iterator.minc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 1,
+ table.iterator.scan.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 1,
+ table.security.scan.visibility.default: '',
+ table.split.threshold: 256M
+ }
+ ---
+ scope: TABLE
+ name: accumulo.metadata
+ properties: {
+ table.cache.block.enable: true,
+ table.cache.index.enable: true,
+ table.compaction.major.ratio: 1,
+ table.constraint.1:
org.apache.accumulo.server.constraints.MetadataConstraints,
+ table.durability: sync,
+ table.failures.ignore: false,
+ table.file.compress.blocksize: 32K,
+ table.file.replication: 5,
+ table.group.server: 'file,log,srv,future',
+ table.group.tablet: '~tab,loc',
+ table.groups.enabled: 'tablet,server',
+ table.iterator.majc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 1,
+ table.iterator.minc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 1,
+ table.iterator.scan.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 1,
+ table.security.scan.visibility.default: '',
+ table.split.threshold: 1234K
+ }
+ ---
+ scope: TABLE
+ name: accumulo.root
+ properties: {
+ table.cache.block.enable: true,
+ table.cache.index.enable: true,
+ table.compaction.major.ratio: 1,
+ table.constraint.1:
org.apache.accumulo.server.constraints.MetadataConstraints,
+ table.durability: sync,
+ table.failures.ignore: false,
+ table.file.compress.blocksize: 32K,
+ table.file.replication: 5,
+ table.group.server: 'file,log,srv,future',
+ table.group.tablet: '~tab,loc',
+ table.groups.enabled: 'tablet,server',
+ table.iterator.majc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 1,
+ table.iterator.minc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 1,
+ table.iterator.scan.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 1,
+ table.security.scan.visibility.default: '',
+ table.split.threshold: 64M
+ }
+ ---
+ scope: TABLE
+ name: accumulo.scanref
+ properties: {
+ table.cache.block.enable: true,
+ table.cache.index.enable: true,
+ table.compaction.major.ratio: 1,
+ table.durability: sync,
+ table.failures.ignore: false,
+ table.file.compress.blocksize: 32K,
+ table.file.replication: 5,
+ table.iterator.majc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 1,
+ table.iterator.minc.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 1,
+ table.iterator.scan.vers:
'10,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 1,
+ table.security.scan.visibility.default: ''
+ }
+ ---
+ scope: TABLE
+ name: production.customers
+ properties: {
+ table.constraint.1:
org.apache.accumulo.core.data.constraints.DefaultKeySizeConstraint,
+ table.file.max: 7,
+ table.iterator.majc.filter: '200,WorkingFilter.class',
+ table.iterator.majc.filter.opt.dropMissing: false,
+ table.iterator.majc.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 1,
+ table.iterator.minc.filter: '200,WorkingFilter.class',
+ table.iterator.minc.filter.opt.dropMissing: false,
+ table.iterator.minc.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 1,
+ table.iterator.scan.filter: '200,WorkingFilter.class',
+ table.iterator.scan.filter.opt.dropMissing: false,
+ table.iterator.scan.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 1
+ }
+ ---
+ scope: TABLE
+ name: production.payments
+ properties: {
+ table.cache.block.enable: true,
+ table.constraint.1:
org.apache.accumulo.core.data.constraints.DefaultKeySizeConstraint,
+ table.file.compress.blocksize.index: 32K,
+ table.iterator.majc.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 1,
+ table.iterator.minc.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 1,
+ table.iterator.scan.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 1
+ }
+ ---
+ scope: TABLE
+ name: testing.customers
+ properties: {
+ table.iterator.majc.betaFilter: '200,ExperimentalFilter.class',
+ table.iterator.majc.betaFilter.opt.dropMissing: true,
+ table.iterator.minc.betaFilter: '200,ExperimentalFilter.class',
+ table.iterator.minc.betaFilter.opt.dropMissing: true,
+ table.iterator.scan.betaFilter: '200,ExperimentalFilter.class',
+ table.iterator.scan.betaFilter.opt.dropMissing: true
+ }
+ ---
+ scope: TABLE
+ name: testing.payments
+ properties: {
+ table.iterator.majc.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.majc.vers.opt.maxVersions: 10,
+ table.iterator.minc.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.minc.vers.opt.maxVersions: 10,
+ table.iterator.scan.vers:
'20,org.apache.accumulo.core.iterators.user.VersioningIterator',
+ table.iterator.scan.vers.opt.maxVersions: 10
+ }
+ """;
+
+ private final static Map<String,String> SYSTEM_PROPS;
+ private final static Map<ResourceGroupId,Map<String,String>> RG_PROPS;
+ private final static Map<String,Map<String,String>> NAMESPACE_PROPS;
+
+ record TableChanges(String name, boolean create, boolean noDefaults,
List<IteratorSetting> iters,
+ Map<String,String> props, Set<String> delete) {
+ }
+
+ private final static List<TableChanges> TABLE_CHANGES;
+
+ private final static ImportConfigCommand.Opts DEFAULT_OPTS = new
ImportConfigCommand.Opts();
+ private final static ImportConfigCommand.Opts IGNORE_EXTRA_OPTS = new
ImportConfigCommand.Opts();
+ private final static ImportConfigCommand.Opts DRY_RUN_IGNORE_EXTRA_OPTS =
+ new ImportConfigCommand.Opts();
+ private final static ImportConfigCommand.Opts DRY_RUN_OPTS = new
ImportConfigCommand.Opts();
+ static {
+ var systemProps = new HashMap<String,String>();
+
+ var testCompactionService = """
+ [{"group":"test_small_compactors",
"maxSize":"256M"},{"group":"test_large_compactors"}]
+ """;
+ var prodCompactionService = """
+ [{"group":"prod_small_compactors",
"maxSize":"256M"},{"group":"prod_large_compactors"}]
+ """;
+ var prodMaintenanceService = """
+ [{"group":"prod_maintenance_compactors"}]
+ """;
+ systemProps.put("compaction.service.prod.planner",
+ "org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner");
+ systemProps.put("compaction.service.prod.planner.opts.groups",
prodCompactionService);
+ systemProps.put("compaction.service.maintenance.planner",
+ "org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner");
+ systemProps.put("compaction.service.maintenance.planner.opts.groups",
prodMaintenanceService);
+ systemProps.put("compaction.service.test.planner",
+ "org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner");
+ systemProps.put("compaction.service.test.planner.opts.groups",
testCompactionService);
+
+ SYSTEM_PROPS = Map.copyOf(systemProps);
+
+ var rgProps = new HashMap<ResourceGroupId,Map<String,String>>();
+ var prodProps = new HashMap<String,String>();
+ prodProps.put(Property.TSERV_DATACACHE_SIZE.getKey(), "32G");
+ prodProps.put(Property.TSERV_INDEXCACHE_SIZE.getKey(), "4G");
+ prodProps.put(Property.SSERV_DATACACHE_SIZE.getKey(), "16G");
+ prodProps.put(Property.SSERV_INDEXCACHE_SIZE.getKey(), "2G");
+ // Try setting special chars and ensure they make it through encoding
to/from yaml
+ prodProps.put(Property.GENERAL_PREFIX + "special.chars",
+ "!@#$%^&*()_\n+-=<,>.?/\t:;\"''{[}]\\~`|");
+
+ rgProps.put(ResourceGroupId.of("prod"), Map.copyOf(prodProps));
+
+ var testProps = new HashMap<String,String>();
+ testProps.put(Property.TSERV_DATACACHE_SIZE.getKey(), "2G");
+ testProps.put(Property.TSERV_INDEXCACHE_SIZE.getKey(), "512M");
+ rgProps.put(ResourceGroupId.of("test"), Map.copyOf(testProps));
+
+ rgProps.put(ResourceGroupId.of("prod_small_compactors"),
+ Map.of(Property.COMPACTOR_FAILURE_TERMINATION_THRESHOLD.getKey(),
"10"));
+ rgProps.put(ResourceGroupId.of("prod_large_compactors"),
+ Map.of(Property.COMPACTOR_FAILURE_TERMINATION_THRESHOLD.getKey(), "5",
+ Property.COMPACTOR_MAX_JOB_WAIT_TIME.getKey(), "3m"));
+ rgProps.put(ResourceGroupId.of("prod_maintenance_compactors"), Map.of());
+
+ rgProps.put(ResourceGroupId.of("test_small_compactors"),
+ Map.of(Property.COMPACTOR_MIN_JOB_WAIT_TIME.getKey(), "10ms",
+ Property.COMPACTOR_MAX_JOB_WAIT_TIME.getKey(), "10s"));
+ rgProps.put(ResourceGroupId.of("test_large_compactors"),
+ Map.of(Property.COMPACTOR_MIN_JOB_WAIT_TIME.getKey(), "10ms",
+ Property.COMPACTOR_MAX_JOB_WAIT_TIME.getKey(), "20s"));
+
+ RG_PROPS = Map.copyOf(rgProps);
+
+ var testingNsProps =
Map.of(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() + "service",
+ "test", "table.custom.assignment.group", "test");
+ var prodNsProps =
Map.of(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() + "service", "prod",
+ Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() + "service.user",
"maintenance",
+ "table.custom.assignment.group", "prod");
+ NAMESPACE_PROPS = Map.of("testing", testingNsProps, "production",
prodNsProps);
+
+ var tableChanges = new ArrayList<TableChanges>();
+
+ var iterSetting1 = new IteratorSetting(200, "betaFilter",
"ExperimentalFilter.class");
+ iterSetting1.addOption("dropMissing", "true");
+ tableChanges.add(new TableChanges("testing.customers", true, true,
List.of(iterSetting1),
+ Map.of(), Set.of()));
+ // remove and change some of the tables default properties
+ tableChanges.add(new TableChanges("testing.payments", true, false,
List.of(),
+ Map.of("table.iterator.majc.vers.opt.maxVersions", "10",
+ "table.iterator.minc.vers.opt.maxVersions", "10",
+ "table.iterator.scan.vers.opt.maxVersions", "10"),
+ Set.of("table.constraint.1")));
+ var iterSetting2 = new IteratorSetting(200, "filter",
"WorkingFilter.class");
+ iterSetting2.addOption("dropMissing", "false");
+ tableChanges.add(new TableChanges("production.customers", true, false,
List.of(iterSetting2),
+ Map.of(Property.TABLE_FILE_MAX.getKey(), "7"), Set.of()));
+ tableChanges.add(new TableChanges("production.payments", true, false,
List.of(),
+ Map.of(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true",
+ Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), "32K"),
+ Set.of()));
+ // override a metadata table property
+ tableChanges.add(new TableChanges("accumulo.metadata", false, false,
List.of(),
+ Map.of(Property.TABLE_SPLIT_THRESHOLD.getKey(), "1234K"), Set.of()));
+ TABLE_CHANGES = List.copyOf(tableChanges);
+
+ IGNORE_EXTRA_OPTS.ignoreExtra = true;
+ DRY_RUN_IGNORE_EXTRA_OPTS.dryRun = true;
+ DRY_RUN_IGNORE_EXTRA_OPTS.ignoreExtra = true;
+ DRY_RUN_OPTS.dryRun = true;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
+ // Processes are killed so set this to make WALs works
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test
+ public void testExport() throws Exception {
+
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+
+ for (var entry : SYSTEM_PROPS.entrySet()) {
+ client.instanceOperations().setProperty(entry.getKey(),
entry.getValue());
+ }
+
+ for (var entry : RG_PROPS.entrySet()) {
+ client.resourceGroupOperations().create(entry.getKey());
+ for (var pe : entry.getValue().entrySet()) {
+ client.resourceGroupOperations().setProperty(entry.getKey(),
pe.getKey(), pe.getValue());
+ }
+ }
+
+ for (var entry : NAMESPACE_PROPS.entrySet()) {
+ client.namespaceOperations().create(entry.getKey());
+ for (var pe : entry.getValue().entrySet()) {
+ client.namespaceOperations().setProperty(entry.getKey(),
pe.getKey(), pe.getValue());
+ }
+ }
+
+ for (var tableProps : TABLE_CHANGES) {
+ if (tableProps.create()) {
+ if (tableProps.noDefaults()) {
+ client.tableOperations().create(tableProps.name(),
+ new NewTableConfiguration().withoutDefaults());
+ } else {
+ client.tableOperations().create(tableProps.name);
+ }
+ }
+
+ for (var iter : tableProps.iters()) {
+ client.tableOperations().attachIterator(tableProps.name(), iter);
+ }
+
+ for (var entry : tableProps.props().entrySet()) {
+ client.tableOperations().setProperty(tableProps.name(),
entry.getKey(), entry.getValue());
+ }
+
+ for (var delProp : tableProps.delete()) {
+ client.tableOperations().removeProperty(tableProps.name(), delProp);
+ }
+ }
+ }
+
+ var yaml = ExportConfigCommand.export(getServerContext());
+ assertEquals(YAML1, yaml);
+ }
+
+ @Test
+ public void testImport() throws Exception {
+
+ Map<String,Map<String,String>> tablePropsBefore = new HashMap<>();
+
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ for (var entry : RG_PROPS.entrySet()) {
+ client.resourceGroupOperations().create(entry.getKey());
+ }
+
+ for (var entry : NAMESPACE_PROPS.entrySet()) {
+ client.namespaceOperations().create(entry.getKey());
+ }
+
+ for (var tableProps : TABLE_CHANGES) {
+ if (tableProps.create()) {
+ client.tableOperations().create(tableProps.name);
+ }
+ }
+
+ for (var table : client.tableOperations().list()) {
+ tablePropsBefore.put(table,
client.tableOperations().getTableProperties(table));
+ }
+ }
+
+ // test a dry run, should not fail and should not change anything
+ try (var in = new ByteArrayInputStream(YAML1.getBytes(UTF_8))) {
+ ImportConfigCommand.load(getServerContext(), in, DRY_RUN_OPTS);
+ }
+
+ // verify dry run did not change anything
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ for (var entry : RG_PROPS.entrySet()) {
+ assertEquals(Map.of(),
client.resourceGroupOperations().getProperties(entry.getKey()));
+ }
+ assertEquals(Map.of(),
+
client.resourceGroupOperations().getProperties(ResourceGroupId.DEFAULT));
+
+ for (var entry : NAMESPACE_PROPS.entrySet()) {
+ assertEquals(Map.of(),
client.namespaceOperations().getNamespaceProperties(entry.getKey()));
+ }
+ assertEquals(Map.of(),
client.namespaceOperations().getNamespaceProperties(""));
+
+ for (var tableProps : TABLE_CHANGES) {
+ assertEquals(tablePropsBefore.get(tableProps.name()),
+ client.tableOperations().getTableProperties(tableProps.name()));
+ }
+ }
+
+ // Now actually do the import
+ try (var in = new ByteArrayInputStream(YAML1.getBytes(UTF_8))) {
+ ImportConfigCommand.load(getServerContext(), in, DEFAULT_OPTS);
+ }
+
+ assertEquals(YAML1, ExportConfigCommand.export(getServerContext()));
+
+ // import export work directly on Zookeeper, ensure these changes are now
visible via the API
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ assertEquals(SYSTEM_PROPS,
client.instanceOperations().getSystemProperties());
+
+ for (var entry : RG_PROPS.entrySet()) {
+ assertEquals(entry.getValue(),
+ client.resourceGroupOperations().getProperties(entry.getKey()));
+ }
+ assertEquals(Map.of(),
+
client.resourceGroupOperations().getProperties(ResourceGroupId.DEFAULT));
+
+ for (var entry : NAMESPACE_PROPS.entrySet()) {
+ assertEquals(entry.getValue(),
+
client.namespaceOperations().getNamespaceProperties(entry.getKey()));
+ }
+ assertEquals(Map.of(),
client.namespaceOperations().getNamespaceProperties(""));
+
+ for (var tableChanges : TABLE_CHANGES) {
+ Map<String,String> expected = new
TreeMap<>(tablePropsBefore.get(tableChanges.name()));
+ if (tableChanges.noDefaults()) {
+
expected.keySet().removeAll(IteratorConfigUtil.getInitialTableProperties().keySet());
+ }
+ expected.keySet().removeAll(tableChanges.delete());
+ expected.putAll(tableChanges.props());
+
+ tableChanges.iters().forEach(setting -> {
+ for (IteratorUtil.IteratorScope scope :
IteratorUtil.IteratorScope.values()) {
+ String root = String.format("%s%s.%s",
Property.TABLE_ITERATOR_PREFIX,
+ scope.name().toLowerCase(), setting.getName());
+ for (Map.Entry<String,String> prop :
setting.getOptions().entrySet()) {
+ expected.put(root + ".opt." + prop.getKey(), prop.getValue());
+ }
+ expected.put(root, setting.getPriority() + "," +
setting.getIteratorClass());
+ }
+ });
+
+ assertEquals(expected,
+ new
TreeMap<>(client.tableOperations().getTableProperties(tableChanges.name())));
+ }
+
+ // Test importing into a single table
+ String singleTableImport = """
+ scope: TABLE
+ name: testing.customers
+ properties: {
+ table.iterator.scan.betaFilter: '150,ExperimentalFilter.class',
+ table.iterator.scan.betaFilter.opt.dropMissing: 'false'
+ }
+ """;
+ try (var in = new
ByteArrayInputStream(singleTableImport.getBytes(UTF_8))) {
+ ImportConfigCommand.load(getServerContext(), in, IGNORE_EXTRA_OPTS);
+ }
+ assertEquals(
+ Map.of("table.iterator.scan.betaFilter",
"150,ExperimentalFilter.class",
+ "table.iterator.scan.betaFilter.opt.dropMissing", "false"),
+ client.tableOperations().getTableProperties("testing.customers"));
+
+ Set<String> tablesNotChanged = new
HashSet<>(client.tableOperations().list());
+ TABLE_CHANGES.forEach(tc -> tablesNotChanged.remove(tc.name()));
+ for (var table : tablesNotChanged) {
+ assertEquals(tablePropsBefore.get(table),
+ client.tableOperations().getTableProperties(table));
+ }
+
+ }
+ }
+
+ @Test
+ public void testMismatch() throws Exception {
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ Map<String,Map<String,String>> tablePropsBefore = new HashMap<>();
+ for (var entry : RG_PROPS.entrySet()) {
+ client.resourceGroupOperations().create(entry.getKey());
+ }
+
+ for (var entry : NAMESPACE_PROPS.entrySet()) {
+ client.namespaceOperations().create(entry.getKey());
+ }
+
+ for (var tableProps : TABLE_CHANGES) {
+ if (tableProps.create()) {
+ client.tableOperations().create(tableProps.name);
+ }
+ }
+ for (var table : client.tableOperations().list()) {
+ tablePropsBefore.put(table,
client.tableOperations().getTableProperties(table));
+ }
+
+ for (var opts : List.of(DRY_RUN_OPTS, DEFAULT_OPTS)) {
+ // delete a RG, should cause import to fail
+ var rgid = RG_PROPS.keySet().iterator().next();
+ client.resourceGroupOperations().remove(rgid);
+ assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(YAML1.getBytes(UTF_8)), opts);
+ });
+ client.resourceGroupOperations().create(rgid);
+
+ // delete a table
+ var table = TABLE_CHANGES.get(0).name();
+ client.tableOperations().delete(table);
+ assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(YAML1.getBytes(UTF_8)), opts);
+ });
+ client.tableOperations().create(table);
+
+ // add an extra rg not in the yaml
+
client.resourceGroupOperations().create(ResourceGroupId.of("g123456789"));
+ assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(YAML1.getBytes(UTF_8)), opts);
+ });
+
client.resourceGroupOperations().remove(ResourceGroupId.of("g123456789"));
+
+ // add an extra namespace not in the yaml
+ client.namespaceOperations().create("ns123456789");
+ assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(YAML1.getBytes(UTF_8)), opts);
+ });
+ client.namespaceOperations().delete("ns123456789");
+
+ // add an extra table not in the yaml
+ client.tableOperations().create("ns123456789");
+ assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(YAML1.getBytes(UTF_8)), opts);
+ });
+ client.tableOperations().delete("ns123456789");
+ }
+ // ensure nothing was changed by the failed imports
+ for (var rgid2 : RG_PROPS.keySet()) {
+ assertEquals(Map.of(),
client.resourceGroupOperations().getProperties(rgid2));
+ }
+
+ for (var ns : NAMESPACE_PROPS.keySet()) {
+ assertEquals(Map.of(),
client.namespaceOperations().getNamespaceProperties(ns));
+ }
+
+ for (var table2 : client.tableOperations().list()) {
+ assertEquals(tablePropsBefore.get(table2),
+ client.tableOperations().getTableProperties(table2));
+ }
+
+ assertNotEquals(YAML1, ExportConfigCommand.export(getServerContext()));
+ // import should succeed now
+ ImportConfigCommand.load(getServerContext(), new
ByteArrayInputStream(YAML1.getBytes(UTF_8)),
+ DEFAULT_OPTS);
+ assertEquals(YAML1, ExportConfigCommand.export(getServerContext()));
+
+ // create some extra stuff that is not in the yaml and try importing w/
allowExtra option
+
client.resourceGroupOperations().create(ResourceGroupId.of("g123456789"));
+ client.namespaceOperations().create("ns123456789");
+ client.tableOperations().create("table123456789");
+ ImportConfigCommand.load(getServerContext(), new
ByteArrayInputStream(YAML1.getBytes(UTF_8)),
+ IGNORE_EXTRA_OPTS);
+ // the exported yaml should have the extra stuff in it, so it should no
longer be equals
+ var newExport = ExportConfigCommand.export(getServerContext());
+ assertNotEquals(YAML1, newExport);
+ assertTrue(newExport.contains("g123456789"));
+ assertTrue(newExport.contains("ns123456789"));
+ assertTrue(newExport.contains("table123456789"));
+
+ }
+ }
+
+ @Test
+ public void testInvalid() throws Exception {
+ // Ensure property validation runs on import and check that it runs for
each scope.
+
+ for (var opts : List.of(IGNORE_EXTRA_OPTS, DRY_RUN_IGNORE_EXTRA_OPTS)) {
+
+ var tablePropsInSystem = """
+ scope: SYSTEM
+ name: ''
+ properties: {
+ table.cache.block.enable: 'true',
+ table.cache.index.enable: 'true'
+ }
+ """;
+
+ var iae = assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(tablePropsInSystem.getBytes(UTF_8)),
opts);
+ });
+ assertTrue(iae.getMessage().contains("Set table properties at the
namespace or table level"),
+ iae::getMessage);
+
+ var tablePropsInRG = """
+ scope: RESOURCE_GROUP
+ name: 'default'
+ properties: {
+ table.cache.block.enable: 'true',
+ table.cache.index.enable: 'true'
+ }
+ """;
+
+ iae = assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(tablePropsInRG.getBytes(UTF_8)), opts);
+ });
+ assertTrue(iae.getMessage().contains("Set table properties at the
namespace or table level"),
+ iae::getMessage);
+
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ client.namespaceOperations().create("ievNS");
+ client.tableOperations().create("ievNS.ievTable",
+ new NewTableConfiguration().withoutDefaults());
+ }
+
+ var invalidNamespaceProps = """
+ scope: NAMESPACE
+ name: ievNS
+ properties: {
+ table.split.threshold: 'abc'
+ }
+ """;
+
+ iae = assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(invalidNamespaceProps.getBytes(UTF_8)),
opts);
+ });
+ assertTrue(iae.getMessage().contains("table.split.threshold"),
iae::getMessage);
+
+ var invalidTableProps = """
+ scope: TABLE
+ name: ievNS.ievTable
+ properties: {
+ table.split.threshold: 'abc'
+ }
+ """;
+
+ iae = assertThrows(IllegalArgumentException.class, () -> {
+ ImportConfigCommand.load(getServerContext(),
+ new ByteArrayInputStream(invalidTableProps.getBytes(UTF_8)), opts);
+ });
+ assertTrue(iae.getMessage().contains("table.split.threshold"),
iae::getMessage);
+
+ // ensure nothing changed
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ assertEquals(Map.of(),
client.instanceOperations().getSystemProperties());
+ assertEquals(Map.of(),
+
client.resourceGroupOperations().getProperties(ResourceGroupId.DEFAULT));
+ assertEquals(Map.of(),
client.namespaceOperations().getNamespaceProperties("ievNS"));
+ assertEquals(Map.of(),
client.tableOperations().getTableProperties("ievNS.ievTable"));
+
+ client.tableOperations().delete("ievNS.ievTable");
+ client.namespaceOperations().delete("ievNS");
+ }
+ }
+
+ }
+
+ @Test
+ public void testOffline() throws Exception {
+ // Ensures import/export can work when no accumulo server processes are
running
+
+ // create a table and set some props
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ var props = Map.of("table.split.threshold", "12345M");
+ client.tableOperations().create("ieOffline",
+ new NewTableConfiguration().withoutDefaults().setProperties(props));
+ }
+
+ // stop all the servers
+ for (var serverType : ServerType.values()) {
+ if (serverType == ServerType.ZOOKEEPER) {
+ continue;
+ }
+ getClusterControl().stopAllServers(serverType);
+ }
+
+ // Do export, edit, import
+ var yaml = ExportConfigCommand.export(getServerContext());
+ assertTrue(yaml.contains("ieOffline") &&
yaml.contains("table.split.threshold: 12345M"), yaml);
+ var newYaml = yaml.replace("table.split.threshold: 12345M",
"table.split.threshold: 1234M");
+
+ try (var in = new ByteArrayInputStream(newYaml.getBytes(UTF_8))) {
+ ImportConfigCommand.load(getServerContext(), in, DEFAULT_OPTS);
+ }
+
+ // restart the servers
+ for (var serverType : ServerType.values()) {
+ getClusterControl().startAllServers(serverType);
+ }
+
+ // verify the tables props were updated
+ try (var client = Accumulo.newClient().from(getClientProps()).build()) {
+ var expected = Map.of("table.split.threshold", "1234M");
+ assertEquals(expected,
client.tableOperations().getTableProperties("ieOffline"));
+ }
+ }
+}
