This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 1fd1f797ca improve scan output checks with unique values (#3814)
1fd1f797ca is described below
commit 1fd1f797ca70f043992fd90485c9dcb33641e928
Author: EdColeman <d...@etcoleman.com>
AuthorDate: Thu Oct 5 14:21:14 2023 -0400
improve scan output checks with unique values (#3814)
* improve scan output checks with unique values
Closes #3806
Co-authored-by: Keith Turner <ktur...@apache.org>
---
.../org/apache/accumulo/test/shell/MockShell.java | 17 ++---
.../apache/accumulo/test/shell/ShellServerIT.java | 81 ++++++++++++++--------
2 files changed, 57 insertions(+), 41 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
b/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
index 1037733391..a8ffc850d3 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
@@ -40,8 +40,8 @@ public class MockShell {
private static final Logger shellLog =
LoggerFactory.getLogger(MockShell.class);
private static final ErrorMessageCallback noop = new ErrorMessageCallback();
- public TestOutputStream output;
- public StringInputStream input;
+ final TestOutputStream output;
+ final StringInputStream input;
public Shell shell;
public LineReader reader;
public Terminal terminal;
@@ -70,18 +70,17 @@ public class MockShell {
shell.setExit(false);
}
- String exec(String cmd) throws IOException {
+ String exec(String cmd) {
output.clear();
shell.execCommand(cmd, true, true);
return output.get();
}
- String exec(String cmd, boolean expectGoodExit) throws IOException {
+ String exec(String cmd, boolean expectGoodExit) {
return exec(cmd, expectGoodExit, noop);
}
- String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback
callback)
- throws IOException {
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback
callback) {
String result = exec(cmd);
if (expectGoodExit) {
assertGoodExit("", true, callback);
@@ -106,7 +105,7 @@ public class MockShell {
}
String exec(String cmd, boolean expectGoodExit, String expectString, boolean
stringPresent,
- ErrorMessageCallback callback) throws IOException {
+ ErrorMessageCallback callback) {
String result = exec(cmd);
if (expectGoodExit) {
assertGoodExit(expectString, stringPresent, callback);
@@ -116,10 +115,6 @@ public class MockShell {
return result;
}
- void assertGoodExit(String s, boolean stringPresent) {
- assertGoodExit(s, stringPresent, noop);
- }
-
void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback
callback) {
shellLog.debug("Shell Output: '{}'", output.get());
if (shell.getExitCode() != 0) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index 1122b13338..59805e5118 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@ -36,9 +36,11 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
+import java.security.SecureRandom;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Base64;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -114,6 +116,8 @@ public class ShellServerIT extends SharedMiniClusterBase {
private static String rootPath;
+ private final SecureRandom random = new SecureRandom();
+
private static class ShellServerITConfigCallback implements
MiniClusterConfigurationCallback {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
coreSite) {
@@ -288,7 +292,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void egrep() throws Exception {
+ public void egrep() {
final String table = getUniqueNames(1)[0];
// egrep
@@ -441,7 +445,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void setIterOptionPrompt() throws Exception {
+ public void setIterOptionPrompt() {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
final String[] tableNames = getUniqueNames(4);
final String tableName0 = tableNames[0];
@@ -520,7 +524,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
protected void checkTableForProperty(final AccumuloClient client, final
String tableName,
- final String expectedKey, final String expectedValue) throws Exception {
+ final String expectedKey, final String expectedValue) {
Wait.waitFor(
() ->
client.tableOperations().getConfiguration(tableName).get(expectedKey)
.equals(expectedValue),
@@ -631,7 +635,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void byeQuitExit() throws Exception {
+ public void byeQuitExit() {
// bye, quit, exit
for (String cmd : "bye quit exit".split(" ")) {
assertFalse(ts.shell.getExit());
@@ -912,7 +916,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void testCompactionSelectionAndStrategy() throws Exception {
+ public void testCompactionSelectionAndStrategy() {
final String table = getUniqueNames(1)[0];
@@ -1146,7 +1150,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void formatter() throws Exception {
+ public void formatter() {
ts.exec("createtable formatter_test", true);
ts.exec("table formatter_test", true);
ts.exec("insert row cf cq 1234abcd", true);
@@ -1336,11 +1340,13 @@ public class ShellServerIT extends
SharedMiniClusterBase {
File errorsDir = new File(rootPath, "errors_" + table);
assertTrue(errorsDir.mkdir());
fs.mkdirs(new Path(errorsDir.toString()));
- File importDir = createRFiles(conf, fs, table);
+
+ String nonce = generateNonce();
+ File importDir = createRFiles(conf, fs, table, nonce);
ts.exec("createtable " + table, true);
ts.exec("importdirectory " + importDir + " " + errorsDir + " true", true);
- ts.exec("scan -r 00000000", true, "00000000", true);
- ts.exec("scan -r 00000099", true, "00000099", true);
+ ts.exec("scan -r 00000000", true, "0-->" + nonce, true);
+ ts.exec("scan -r 00000099", true, "99-->" + nonce, true);
ts.exec("deletetable -f " + table);
}
@@ -1348,12 +1354,14 @@ public class ShellServerIT extends
SharedMiniClusterBase {
public void importDirectory() throws Exception {
final String table = getUniqueNames(1)[0];
Configuration conf = new Configuration();
+ String nonce = generateNonce();
FileSystem fs = FileSystem.get(conf);
- File importDir = createRFiles(conf, fs, table);
+ File importDir = createRFiles(conf, fs, table, nonce);
ts.exec("createtable " + table, true);
ts.exec("importdirectory " + importDir + " true", true);
- ts.exec("scan -r 00000000", true, "00000000", true);
- ts.exec("scan -r 00000099", true, "00000099", true);
+ ts.exec("scan -r 00000000", true, "0-->" + nonce, true);
+ ts.exec("scan -r 00000099", true, "99-->" + nonce, true);
+ ts.exec("scan -r 00000099", true, "99-->" + nonce + nonce, false);
ts.exec("deletetable -f " + table);
}
@@ -1362,25 +1370,27 @@ public class ShellServerIT extends
SharedMiniClusterBase {
public void importDirectoryWithOptions() throws Exception {
final String table = getUniqueNames(1)[0];
Configuration conf = new Configuration();
+ String nonce = generateNonce();
FileSystem fs = FileSystem.get(conf);
- File importDir = createRFiles(conf, fs, table);
+ File importDir = createRFiles(conf, fs, table, nonce);
ts.exec("createtable " + table, true);
ts.exec("notable", true);
ts.exec("importdirectory -t " + table + " -i " + importDir + " true",
true);
- ts.exec("scan -t " + table + " -r 00000000", true, "00000000", true);
- ts.exec("scan -t " + table + " -r 00000099", true, "00000099", true);
+ ts.exec("scan -t " + table + " -np -b 0 -e 2", true, "0-->" + nonce, true);
+ ts.exec("scan -t " + table + " -b 00000098 -e 00000100", true, "99-->" +
nonce, true);
// Attempt to re-import without -i option, error should occur
ts.exec("importdirectory -t " + table + " " + importDir + " true", false);
// Attempt re-import once more, this time with -i option. No error should
occur, only a
// message indicating the directory was empty and zero files were imported
ts.exec("importdirectory -t " + table + " -i " + importDir + " true",
true);
- ts.exec("scan -t " + table + " -r 00000000", true, "00000000", true);
- ts.exec("scan -t " + table + " -r 00000099", true, "00000099", true);
+ ts.exec("scan -t " + table + " -r 00000000", true, "0-->" + nonce, true);
+ ts.exec("scan -t " + table + " -r 00000099", true, "99-->" + nonce, true);
+ ts.exec("scan -t " + table + " -r 00000201", true, nonce, false);
ts.exec("deletetable -f " + table);
}
- private File createRFiles(final Configuration conf, final FileSystem fs,
final String postfix)
- throws IOException {
+ private File createRFiles(final Configuration conf, final FileSystem fs,
final String postfix,
+ final String nonce) throws IOException {
File importDir = new File(rootPath, "import_" + postfix);
assertTrue(importDir.mkdir());
String even = new File(importDir, "even.rf").toString();
@@ -1395,11 +1405,13 @@ public class ShellServerIT extends
SharedMiniClusterBase {
long timestamp = System.currentTimeMillis();
Text cf = new Text("cf");
Text cq = new Text("cq");
- Value value = new Value("value");
+ Value value;
for (int i = 0; i < 100; i += 2) {
- Key key = new Key(new Text(String.format("%8d", i)), cf, cq, timestamp);
+ Key key = new Key(new Text(String.format("%08d", i)), cf, cq, timestamp);
+ value = new Value(i + "-->" + nonce);
evenWriter.append(key, value);
- key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, timestamp);
+ key = new Key(new Text(String.format("%08d", i + 1)), cf, cq, timestamp);
+ value = new Value(i + 1 + "-->" + nonce);
oddWriter.append(key, value);
}
evenWriter.close();
@@ -1533,7 +1545,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void systempermission() throws Exception {
+ public void systempermission() {
ts.exec("systempermissions");
assertEquals(12, ts.output.get().split("\n").length - 1);
ts.exec("tablepermissions", true);
@@ -1666,7 +1678,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void badLogin() throws Exception {
+ public void badLogin() {
// Can't run with Kerberos, can't switch identity in shell presently
assumeTrue(getToken() instanceof PasswordToken);
ts.input.set(getRootPassword() + "\n");
@@ -1758,7 +1770,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
ts.exec("insert r cf cq abc", true);
}
- private int countkeys(String table) throws IOException {
+ private int countkeys(String table) {
ts.exec("scan -np -t " + table);
return ts.output.get().split("\n").length - 1;
}
@@ -2025,13 +2037,13 @@ public class ShellServerIT extends
SharedMiniClusterBase {
}
}
- private void make10() throws IOException {
+ private void make10() {
for (int i = 0; i < 10; i++) {
ts.exec(String.format("insert row%d cf col%d value", i, i));
}
}
- private List<String> getFiles(String tableId) throws IOException {
+ private List<String> getFiles(String tableId) {
ts.output.clear();
ts.exec(
@@ -2049,7 +2061,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
}
- private int countFiles(String tableId) throws IOException {
+ private int countFiles(String tableId) {
return getFiles(tableId).size();
}
@@ -2082,7 +2094,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void testSummaries() throws Exception {
+ public void testSummaries() {
String tableName = getUniqueNames(1)[0];
ts.exec("createtable " + tableName);
ts.exec(
@@ -2152,7 +2164,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
}
@Test
- public void testSummarySelection() throws Exception {
+ public void testSummarySelection() {
String tableName = getUniqueNames(1)[0];
ts.exec("createtable " + tableName);
// will create a few files and do not want them compacted
@@ -2282,4 +2294,13 @@ public class ShellServerIT extends SharedMiniClusterBase
{
}
+ /**
+ * Generates a string based on a random number that can be used to uniquely
identify data in shell
+ * output
+ */
+ public String generateNonce() {
+ byte[] r = new byte[16];
+ random.nextBytes(r);
+ return new String(Base64.getEncoder().encode(r), UTF_8);
+ }
}
