This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new f4cb86cd67 Added modifyPropertiesFromFile Method in ConfigCommand
(#4096)
f4cb86cd67 is described below
commit f4cb86cd67d7ae728fde63f13a47fa45c0f7c425
Author: rsingh433 <[email protected]>
AuthorDate: Fri Aug 15 09:58:29 2025 -0400
Added modifyPropertiesFromFile Method in ConfigCommand (#4096)
Added modifyPropertiesFromFile Method in ConfigCommand
Updates the createTable command to accept a property file for init
properties.
---------
Co-authored-by: Daniel Roberts ddanielr <[email protected]>
---
shell/pom.xml | 4 ++
.../java/org/apache/accumulo/shell/ShellUtil.java | 39 ++++++++++++++
.../accumulo/shell/commands/ConfigCommand.java | 30 ++++++++++-
.../shell/commands/CreateTableCommand.java | 17 +++++-
.../org/apache/accumulo/test/shell/ShellIT.java | 60 +++++++++++++++++++++-
5 files changed, 145 insertions(+), 5 deletions(-)
diff --git a/shell/pom.xml b/shell/pom.xml
index 6cd66a8297..04c2b6eacc 100644
--- a/shell/pom.xml
+++ b/shell/pom.xml
@@ -71,6 +71,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-configuration2</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
diff --git a/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
b/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
index dd995e0c71..5afcbe66cf 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
@@ -21,19 +21,26 @@ package org.apache.accumulo.shell;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.stream.Collectors;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.Pair;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.hadoop.io.Text;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -74,6 +81,38 @@ public class ShellUtil {
} else {
return Collections.emptyMap();
}
+ }
+
+ public static Map<String,String> readPropertiesFromFile(String filename)
+ throws IOException, AccumuloException {
+ PropertiesConfiguration config = new PropertiesConfiguration();
+ try (FileReader out = new FileReader(filename, UTF_8)) {
+ config.read(out);
+ } catch (ConfigurationException e) {
+ Shell.log.error("Property file {} contains invalid configuration. Please
verify file format",
+ filename, e);
+ }
+ Iterator<String> keysIterator = config.getKeys();
+ Map<String,String> propertiesMap = new HashMap<>();
+ boolean foundErrors = false;
+ while (keysIterator.hasNext()) {
+ String key = keysIterator.next();
+ String value = config.getString(key);
+ if (!Property.isValidPropertyKey(key)) {
+ Shell.log.error("Property: \"{}\" is invalid", key);
+ foundErrors = true;
+ } else if (!Property.isValidProperty(key, value)) {
+ Shell.log.error("Property: \"{}\" has an invalid value: \"{}\"", key,
value);
+ foundErrors = true;
+ } else {
+ propertiesMap.put(key, value);
+ }
+ }
+ if (foundErrors) {
+ Shell.log.error("Property file {} contains invalid properties",
filename);
+ throw new AccumuloException("InvalidPropertyFile: " + filename);
+ }
+ return propertiesMap;
}
}
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ConfigCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ConfigCommand.java
index 30749f143e..7311db5422 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/ConfigCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ConfigCommand.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.shell.commands;
import static
org.apache.accumulo.core.client.security.SecurityErrorCode.PERMISSION_DENIED;
+import static org.apache.accumulo.shell.ShellUtil.readPropertiesFromFile;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,6 +29,7 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
+import java.util.function.Consumer;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -63,6 +65,7 @@ public class ConfigCommand extends Command {
private Option outputFileOpt;
private Option namespaceOpt;
private Option showExpOpt;
+ private Option propFileOpt;
private int COL1 = 10;
private int COL2 = 7;
@@ -100,6 +103,10 @@ public class ConfigCommand extends Command {
&&
!shellState.getAccumuloClient().namespaceOperations().exists(namespace)) {
throw new NamespaceNotFoundException(null, namespace, null);
}
+ String filename = cl.getOptionValue(propFileOpt.getOpt());
+ if (filename != null) {
+ modifyPropertiesFromFile(cl, shellState, filename);
+ }
if (cl.hasOption(deleteOpt.getOpt())) {
// delete property from table, namespace, or system
String property = cl.getOptionValue(deleteOpt.getOpt());
@@ -379,6 +386,25 @@ public class ConfigCommand extends Command {
return 0;
}
+ private void modifyPropertiesFromFile(CommandLine cl, Shell shellState,
String filename)
+ throws AccumuloException, AccumuloSecurityException, IOException,
NamespaceNotFoundException {
+ Map<String,String> propertiesMap = readPropertiesFromFile(filename);
+
+ Consumer<Map<String,String>> propertyModifier = currProps -> {
+ currProps.putAll(propertiesMap);
+ };
+
+ if (cl.hasOption(tableOpt.getOpt())) {
+ shellState.getAccumuloClient().tableOperations()
+ .modifyProperties(cl.getOptionValue(tableOpt.getOpt()),
propertyModifier);
+ } else if (cl.hasOption(namespaceOpt.getOpt())) {
+ shellState.getAccumuloClient().namespaceOperations()
+ .modifyProperties(cl.getOptionValue(namespaceOpt.getOpt()),
propertyModifier);
+ } else {
+
shellState.getAccumuloClient().instanceOperations().modifyProperties(propertyModifier);
+ }
+ }
+
private boolean matchTheFilterText(CommandLine cl, String key, String value)
{
if (cl.hasOption(filterOpt.getOpt()) &&
!key.contains(cl.getOptionValue(filterOpt.getOpt()))) {
return true;
@@ -437,7 +463,7 @@ public class ConfigCommand extends Command {
outputFileOpt = new Option("o", "output", true, "local file to write the
scan output to");
namespaceOpt = new Option(ShellOptions.namespaceOption, "namespace", true,
"namespace to display/set/delete properties for");
-
+ propFileOpt = new Option("pf", "propFile", true, "file containing
properties to set");
tableOpt.setArgName("table");
deleteOpt.setArgName("property");
setOpt.setArgName("property=value");
@@ -445,11 +471,13 @@ public class ConfigCommand extends Command {
filterWithValuesOpt.setArgName("string");
outputFileOpt.setArgName("file");
namespaceOpt.setArgName("namespace");
+ propFileOpt.setArgName("filename");
og.addOption(deleteOpt);
og.addOption(setOpt);
og.addOption(filterOpt);
og.addOption(filterWithValuesOpt);
+ og.addOption(propFileOpt);
tgroup.addOption(tableOpt);
tgroup.addOption(namespaceOpt);
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
index 9847a8a54c..633b419973 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.shell.commands;
import static org.apache.accumulo.core.util.Validators.NEW_TABLE_NAME;
+import static org.apache.accumulo.shell.ShellUtil.readPropertiesFromFile;
import java.io.IOException;
import java.util.Arrays;
@@ -66,6 +67,7 @@ public class CreateTableCommand extends Command {
private Option base64Opt;
private Option createTableOptFormatter;
private Option createTableOptInitProp;
+ private Option createTableOptInitPropFile;
private Option createTableOptLocalityProps;
private Option createTableOptIteratorProps;
private Option createTableOptOffline;
@@ -120,8 +122,15 @@ public class CreateTableCommand extends Command {
timeType = TimeType.LOGICAL;
}
- Map<String,String> initProperties =
- new HashMap<>(ShellUtil.parseMapOpt(cl, createTableOptInitProp));
+ Map<String,String> initProperties;
+ String filename = cl.getOptionValue(createTableOptInitPropFile.getOpt());
+ if (filename != null) {
+ initProperties = readPropertiesFromFile(filename);
+ } else {
+ initProperties = new HashMap<>();
+ }
+
+ initProperties.putAll(ShellUtil.parseMapOpt(cl, createTableOptInitProp));
// Set iterator if supplied
if (cl.hasOption(createTableOptIteratorProps.getOpt())) {
@@ -327,11 +336,14 @@ public class CreateTableCommand extends Command {
createTableOptFormatter = new Option("f", "formatter", true, "default
formatter to set");
createTableOptInitProp = new Option("prop", "init-properties", true,
"comma-separated user-defined initial key=value pairs");
+ createTableOptInitPropFile =
+ new Option("pf", "propFile", true, "user-defined initial properties
file");
createTableOptCopyConfig.setArgName("table");
createTableOptCopySplits.setArgName("table");
createTableOptSplit.setArgName("filename");
createTableOptFormatter.setArgName("className");
createTableOptInitProp.setArgName("properties");
+ createTableOptInitPropFile.setArgName("properties-file");
createTableOptLocalityProps =
new Option("l", "locality", true, "create locality groups at table
creation");
@@ -368,6 +380,7 @@ public class CreateTableCommand extends Command {
o.addOption(createTableOptEVC);
o.addOption(createTableOptFormatter);
o.addOption(createTableOptInitProp);
+ o.addOption(createTableOptInitPropFile);
o.addOption(createTableOptLocalityProps);
o.addOption(createTableOptIteratorProps);
o.addOption(createTableOptOffline);
diff --git a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
index 27b659a37c..d8fc412d90 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.test.shell;
+import static
org.apache.accumulo.core.conf.Property.TSERV_COMPACTION_WARN_TIME;
import static org.apache.accumulo.harness.AccumuloITBase.MINI_CLUSTER_ONLY;
import static org.apache.accumulo.harness.AccumuloITBase.SUNNY_DAY;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -27,6 +28,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.PrintWriter;
import java.nio.file.Files;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
@@ -40,7 +42,9 @@ import java.util.TimeZone;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.PropertyType;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.shell.Shell;
import org.jline.reader.LineReader;
import org.jline.reader.LineReaderBuilder;
@@ -53,6 +57,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,10 +125,10 @@ public class ShellIT extends SharedMiniClusterBase {
private LineReader reader;
private Terminal terminal;
- void execExpectList(String cmd, boolean expecteGoodExit, List<String>
expectedStrings)
+ void execExpectList(String cmd, boolean expectGoodExit, List<String>
expectedStrings)
throws IOException {
exec(cmd);
- if (expecteGoodExit) {
+ if (expectGoodExit) {
assertGoodExit("", true);
} else {
assertBadExit("", true);
@@ -640,6 +645,57 @@ public class ShellIT extends SharedMiniClusterBase {
assertGoodExit("Unknown command", false);
}
+ @TempDir
+ private static File tempDir;
+
+ @Test
+ public void propFileNotFoundTest() throws IOException {
+ String fileName = new File(tempDir, "propFile.shellit").getAbsolutePath();
+
+ Shell.log.debug("Starting prop file not found test
--------------------------");
+ exec("config --propFile " + fileName, false,
+ "FileNotFoundException: " + fileName + " (No such file or directory)");
+ }
+
+ @Test
+ public void setpropsViaFile() throws Exception {
+ File file = File.createTempFile("propFile", ".conf", tempDir);
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println(TSERV_COMPACTION_WARN_TIME.getKey() + "=11m");
+ writer.close();
+ exec("config --propFile " + file.getAbsolutePath(), true);
+ }
+
+ @Test
+ public void createTableWithPropsFile() throws Exception {
+ File file = File.createTempFile("propFile", ".conf", tempDir);
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("table.custom.test1=true");
+ writer.println("table.custom.test2=false");
+ writer.close();
+ exec("createtable test --propFile " + file.getAbsolutePath()
+ + " -prop table.custom.test3=optional", true);
+
+ assertTrue(shell.getAccumuloClient().tableOperations().exists("test"));
+ Map<String,String> tableIds =
shell.getAccumuloClient().tableOperations().tableIdMap();
+
+ TableConfiguration tableConf =
+
getCluster().getServerContext().getTableConfiguration(TableId.of(tableIds.get("test")));
+ assertEquals("true", tableConf.get("table.custom.test1"));
+ assertEquals("false", tableConf.get("table.custom.test2"));
+ assertEquals("optional", tableConf.get("table.custom.test3"));
+ }
+
+ @Test
+ public void invalidPropFileTest() throws Exception {
+ File file = File.createTempFile("invalidPropFile", ".conf", tempDir);
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("this is not a valid property file");
+ writer.close();
+ exec("config --propFile " + file.getAbsolutePath(), false,
+ "InvalidPropertyFile: " + file.getAbsolutePath());
+ }
+
@Test
public void setIterTest() throws IOException {
Shell.log.debug("Starting setiter test --------------------------");
