This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4b0f6bbb3103286fa0487b4ff101d42e6b22e98d
Merge: f106b8c212 e6026cbdc8
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Thu Aug 17 12:07:00 2023 +0000
Merge branch 'main' into elasticity
assemble/bin/accumulo | 2 +-
assemble/conf/accumulo-env.sh | 9 +-
.../accumulo/core/metrics/MetricsProducer.java | 31 +--
.../its/mapreduce/AccumuloInputFormatIT.java | 2 +-
pom.xml | 2 +-
.../apache/accumulo/monitor/resources/js/navbar.js | 230 ++++++++++++++-------
.../tserver/metrics/TabletServerScanMetrics.java | 2 +-
.../main/java/org/apache/accumulo/shell/Shell.java | 21 ++
.../accumulo/shell/commands/ConfigCommand.java | 32 ++-
.../shell/commands/DeleteNamespaceCommand.java | 14 +-
.../accumulo/shell/commands/DropUserCommand.java | 9 +-
.../accumulo/shell/commands/MergeCommand.java | 11 +-
.../accumulo/shell/commands/TableOperation.java | 24 +--
.../shell/commands/DropUserCommandTest.java | 11 +-
.../apache/accumulo/test/functional/TabletIT.java | 3 +-
.../org/apache/accumulo/test/shell/ShellIT.java | 10 +-
.../apache/accumulo/test/shell/ShellServerIT.java | 55 -----
17 files changed, 239 insertions(+), 229 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 820a432bd2,a2aa7d6a9a..0c66ce5411
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -639,21 -603,10 +639,21 @@@ public interface MetricsProducer
String METRICS_LOW_MEMORY = "accumulo.detected.low.memory";
String METRICS_COMPACTOR_PREFIX = "accumulo.compactor.";
String METRICS_COMPACTOR_MAJC_STUCK = METRICS_COMPACTOR_PREFIX +
"majc.stuck";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUES = METRICS_COMPACTOR_PREFIX +
"queue.count";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH =
METRICS_COMPACTOR_PREFIX + "queue.length";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED =
+ METRICS_COMPACTOR_PREFIX + "queue.jobs.dequeued";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED =
+ METRICS_COMPACTOR_PREFIX + "queue.jobs.queued";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED =
+ METRICS_COMPACTOR_PREFIX + "queue.jobs.rejected";
+
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY =
+ METRICS_COMPACTOR_PREFIX + "queue.jobs.priority";
String METRICS_FATE_PREFIX = "accumulo.fate.";
- String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX +
"ops.in_progress_by_type";
- String METRICS_FATE_TOTAL_IN_PROGRESS = METRICS_FATE_PREFIX +
"ops.in_progress";
+ String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX +
"ops.in.progress.by.type";
+ String METRICS_FATE_TOTAL_IN_PROGRESS = METRICS_FATE_PREFIX +
"ops.in.progress";
String METRICS_FATE_OPS_ACTIVITY = METRICS_FATE_PREFIX + "ops.activity";
String METRICS_FATE_ERRORS = METRICS_FATE_PREFIX + "errors";
String METRICS_FATE_TX = METRICS_FATE_PREFIX + "tx";
diff --cc test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index e5177d8f78,03b6b1c1b2..2c69542334
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@@ -2260,152 -2090,6 +2260,97 @@@ public class ShellServerIT extends Shar
assertMatches(output,
"(?sm).*^.*total[:]2[,]\\s+missing[:]0[,]\\s+extra[:]0.*$.*");
}
+ // This test serves to verify the listtablets command as well as the
getTabletInformation api,
+ // which is used by listtablets.
+ @Test
+ public void testListTablets() throws IOException, InterruptedException {
+
+ final var tables = getUniqueNames(2);
+ final String table1 = tables[0];
+ final String table2 = tables[1];
+
+ ts.exec("createtable " + table1, true);
+ ts.exec("addsplits g n u", true);
+ ts.exec("setgoal -g always -r g", true);
+ ts.exec("setgoal -g always -r u", true);
+ insertData(table1, 1000, 3);
+ ts.exec("compact -w -t " + table1);
+ ts.exec("scan -t " + table1);
+
+ ts.exec("createtable " + table2, true);
+ ts.exec("addsplits f m t", true);
+ ts.exec("setgoal -g always -r n", true);
+ insertData(table2, 500, 5);
+ ts.exec("compact -t " + table2);
+ ts.exec("scan -t " + table1);
+ ts.exec("setgoal -r g -t " + table2 + " -g NEVER");
+
+ // give tablet time to become unassigned
+ for (var i = 0; i < 15; i++) {
+ Thread.sleep(1000);
+ String goal = ts.exec("listtablets -t " + table2, true, "m
NEVER");
+ if (goal.contains("UNASSIGNED None")) {
+ break;
+ }
+ }
+
+ String results = ts.exec("listtablets -np -p
ShellServerIT_testListTablets.", true);
+ assertTrue(results.contains("TABLE: ShellServerIT_testListTablets0"));
+ assertTrue(results.contains("TABLE: ShellServerIT_testListTablets1"));
+ assertTrue(results.contains("1 -INF g
ALWAYS"));
+ assertTrue(results.contains("1 g n
ONDEMAND"));
+ assertTrue(results.contains("1 n u
ALWAYS"));
+ assertTrue(results.contains("1 u +INF
ONDEMAND"));
+ assertTrue(results.contains("2 -INF f
ONDEMAND"));
+ assertTrue(results.contains("2 f m
NEVER"));
+ assertTrue(results.contains("2 m t
ALWAYS"));
+ assertTrue(results.contains("2 t +INF
ONDEMAND"));
+
+ // verify the sum of the tablets sizes, number of entries, and dir name
match the data in a
+ // metadata scan
+ String metadata = ts.exec("scan -np -t accumulo.metadata -b 1 -c
loc,file");
+ for (String line : metadata.split("\n")) {
+ String[] tokens = line.split("\\s+");
+ if (tokens[1].startsWith("loc")) {
+ String loc = tokens[3];
+ assertTrue(results.contains(loc));
+ }
+ if (tokens[1].startsWith("file")) {
+ String[] parts = tokens[1].split("/");
+ String dir = parts[parts.length - 2];
+ assertTrue(results.contains(dir));
+ String[] sizes = tokens[3].split(",");
+ String size = String.format("%,d", Integer.parseInt(sizes[0]));
+ String entries = String.format("%,d", Integer.parseInt(sizes[1]));
+ assertTrue(results.contains(size));
+ assertTrue(results.contains(entries));
+ }
+ }
+ }
+
+ private void insertData(String table, int numEntries, int rowLen) throws
IOException {
+ for (var i = 0; i < numEntries; i++) {
+ String alphabet = "abcdefghijklmnopqrstuvwxyz";
+ String row = String.valueOf(alphabet.charAt(i % 26)) + i;
+ var cf = "cf" + i;
+ var cq = "cq" + i;
+ var data = "asdfqwerty";
+ ts.exec("insert -t " + table + " " + row + " " + cf + " " + cq + " " +
data, true);
+ }
+ }
+
+ private java.nio.file.Path createSplitsFile(final String splitsFile, final
SortedSet<Text> splits)
+ throws IOException {
+ String fullSplitsFile = System.getProperty("user.dir") + "/target/" +
splitsFile;
+ java.nio.file.Path path = Paths.get(fullSplitsFile);
+ try (BufferedWriter writer = Files.newBufferedWriter(path, UTF_8)) {
+ for (Text text : splits) {
+ writer.write(text.toString() + '\n');
+ }
+ }
+ return path;
+ }
+
- @Test
- public void testFateCommandWithSlowCompaction() throws Exception {
- final String table = getUniqueNames(1)[0];
-
- String orgProps = System.getProperty("accumulo.properties");
-
- System.setProperty("accumulo.properties",
- "file://" +
getCluster().getConfig().getAccumuloPropsFile().getCanonicalPath());
- // compact
- ts.exec("createtable " + table);
-
- // setup SlowIterator to sleep for 10 seconds
- ts.exec("config -t " + table
- + " -s
table.iterator.majc.slow=1,org.apache.accumulo.test.functional.SlowIterator");
- ts.exec("config -t " + table + " -s
table.iterator.majc.slow.opt.sleepTime=10000");
-
- // make two files
- ts.exec("insert a1 b c v_a1");
- ts.exec("insert a2 b c v_a2");
- ts.exec("flush -w");
- ts.exec("insert x1 b c v_x1");
- ts.exec("insert x2 b c v_x2");
- ts.exec("flush -w");
-
- // no transactions running
- ts.exec("fate -print", true, "0 transactions", true);
-
- // merge two files into one
- ts.exec("compact -t " + table);
- Thread.sleep(1_000);
- // start 2nd transaction
- ts.exec("compact -t " + table);
- Thread.sleep(3_000);
-
- // 2 compactions should be running so parse the output to get one of the
transaction ids
- log.info("Calling fate print for table = {}", table);
- String result = ts.exec("fate -print", true, "txid:", true);
- String[] resultParts = result.split("txid: ");
- String[] parts = resultParts[1].split(" ");
- String txid = parts[0];
- // test filters
- ts.exec("fate -print -t IN_PROGRESS", true, "2 transactions", true);
- ts.exec("fate -print " + txid + " -t IN_PROGRESS", true, "1
transactions", true);
- ts.exec("fate -print " + txid + " -t FAILED", true, "0 transactions",
true);
- ts.exec("fate -print -t NEW", true, "0 transactions", true);
- ts.exec("fate -print 1234", true, "0 transactions", true);
- ts.exec("fate -print FATE[aaa] 1 2 3", true, "0 transactions", true);
-
- ts.exec("deletetable -f " + table);
-
- if (orgProps != null) {
- System.setProperty("accumulo.properties", orgProps);
- }
- }
-
@Test
public void failOnInvalidClassloaderContestTest() throws Exception {
