This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 370f78bdd2 Remove Java 21-related warnings (#6191)
370f78bdd2 is described below
commit 370f78bdd2a2f74af08909c8f61d965fc8431674
Author: Christopher Tubbs <[email protected]>
AuthorDate: Mon Mar 9 22:20:54 2026 -0400
Remove Java 21-related warnings (#6191)
This commit addresses things that are problems in newer Java versions,
but attempts to get ahead of the issues, to ease the migration to newer
versions. The logging changes are done here at low risk for Accumulo
2.1, since the default logging configuration can be easily changed by
users, and the only affected log message formats in code are trace
level and not typically seen by most users.
* Remove logging uses of `Thread.getId()` in code (deprecated in 21) and
replace with logging config that shows the thread information
* Additional minor changes to `log4j2-test.properties` files to make it
easier to compare them and keep them in sync
* Add Threads.toString(Thread) method for one case where logging more
information about the thread than just the Thread ID would be useful
(in TabletServerBatchWriter.java)
* Replace use of `new URL(String)` (deprecated in 21) with
`URI.create(String).toURL()`
* Remove unneeded MiniDFSUtil that was using a deprecated exec method;
this code is not needed because Hadoop already sets a suitable default
value for the `dfs.datanode.data.dir.perm` and we don't need to
compute a different one using this code; similarly remove the
unnecessary `hadoop.security.token.service.use_ip` property which was
just setting the default value.
* Remove unused `CollectTabletStats` test-only class and its
corresponding test case, since it is not used by any test code (this
removes another case where the deprecated `Thread.getId()` was being
called.
* Add relative path to parent POM (best practice)
---
assemble/conf/log4j2-service.properties | 7 +-
assemble/conf/log4j2.properties | 3 +-
assemble/pom.xml | 1 +
core/pom.xml | 1 +
.../accumulo/core/client/ZooKeeperInstance.java | 16 +-
.../accumulo/core/clientImpl/ClientContext.java | 10 +-
.../core/clientImpl/NamespaceOperationsImpl.java | 11 +-
.../core/clientImpl/RootTabletLocator.java | 5 +-
.../core/clientImpl/TableOperationsImpl.java | 13 +-
.../core/clientImpl/TabletLocatorImpl.java | 24 +-
.../TabletServerBatchReaderIterator.java | 20 +-
.../core/clientImpl/TabletServerBatchWriter.java | 7 +-
.../accumulo/core/clientImpl/ThriftScanner.java | 16 +-
.../core/metadata/MetadataLocationObtainer.java | 8 +-
.../apache/accumulo/core/util/threads/Threads.java | 15 +
.../core/classloader/URLClassLoaderFactory.java | 6 +-
core/src/test/resources/log4j2-test.properties | 5 +-
hadoop-mapreduce/pom.xml | 1 +
.../src/test/resources/log4j2-test.properties | 5 +-
iterator-test-harness/pom.xml | 1 +
.../src/test/resources/log4j2-test.properties | 5 +-
minicluster/pom.xml | 1 +
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 2 -
.../src/test/resources/log4j2-test.properties | 5 +-
pom.xml | 1 +
.../base/src/test/resources/log4j2-test.properties | 4 +-
.../src/test/resources/log4j2-test.properties | 8 +-
.../src/test/resources/log4j2-test.properties | 8 +-
.../gc/src/test/resources/log4j2-test.properties | 5 +-
.../src/test/resources/log4j2-test.properties | 5 +-
.../java/org/apache/accumulo/monitor/Monitor.java | 10 +-
.../src/test/resources/log4j2-test.properties | 5 +-
.../src/test/resources/log4j2-test.properties | 5 +-
shell/pom.xml | 1 +
shell/src/test/resources/log4j2-test.properties | 5 +-
start/pom.xml | 1 +
.../start/classloader/vfs/MiniDFSUtil.java | 53 --
.../accumulo/start/test/AccumuloDFSBase.java | 4 -
start/src/test/resources/log4j2-test.properties | 5 +-
test/pom.xml | 1 +
.../accumulo/test/CountNameNodeOpsBulkIT.java | 4 +-
.../ThriftServerBindsBeforeZooKeeperLockIT.java | 4 +-
.../accumulo/test/functional/MonitorSslIT.java | 4 +-
.../accumulo/test/functional/ReadWriteIT.java | 4 +-
.../test/performance/scan/CollectTabletStats.java | 615 ---------------------
test/src/main/resources/log4j2-test.properties | 5 +-
.../performance/scan/CollectTabletStatsTest.java | 52 --
47 files changed, 133 insertions(+), 864 deletions(-)
diff --git a/assemble/conf/log4j2-service.properties
b/assemble/conf/log4j2-service.properties
index a661d84fc1..f96e472252 100644
--- a/assemble/conf/log4j2-service.properties
+++ b/assemble/conf/log4j2-service.properties
@@ -32,7 +32,7 @@ appender.console.type = Console
appender.console.name = STDERR
appender.console.target = SYSTEM_ERR
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T] [%-8c{2}] %-5p: %m%n
appender.console.filter.threshold.type = ThresholdFilter
appender.console.filter.threshold.level = info
@@ -41,7 +41,7 @@ appender.rolling.name = LogFiles
appender.rolling.fileName = ${filename}.log
appender.rolling.filePattern = ${filename}-%d{yyyy-MM-dd}-%i.log.gz
appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.rolling.layout.pattern = %d{ISO8601} Thread[%T] [%-8c{2}] %-5p: %m%n
appender.rolling.policies.type = Policies
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval = 1
@@ -56,7 +56,7 @@ appender.audit.name = AuditLogFiles
appender.audit.fileName = ${filename}.audit
appender.audit.filePattern = ${filename}-%d{yyyy-MM-dd}-%i.audit.gz
appender.audit.layout.type = PatternLayout
-appender.audit.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.audit.layout.pattern = %d{ISO8601} Thread[%T] [%-8c{2}] %-5p: %m%n
appender.audit.policies.type = Policies
appender.audit.policies.time.type = TimeBasedTriggeringPolicy
appender.audit.policies.time.interval = 1
@@ -90,4 +90,3 @@ rootLogger.level = info
rootLogger.appenderRef.console.ref = STDERR
rootLogger.appenderRef.rolling.ref = LogFiles
rootLogger.appenderRef.monitor.ref = MonitorLog
-
diff --git a/assemble/conf/log4j2.properties b/assemble/conf/log4j2.properties
index 6aafb03645..c216ad1690 100644
--- a/assemble/conf/log4j2.properties
+++ b/assemble/conf/log4j2.properties
@@ -27,7 +27,7 @@ appender.console.type = Console
appender.console.name = STDERR
appender.console.target = SYSTEM_ERR
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %style{%d{ISO8601}}{dim,cyan}
%style{[}{red}%style{%-8c{2}}{dim,blue}%style{]}{red}
%highlight{%-5p}%style{:}{red} %m%n
+appender.console.layout.pattern = %style{%d{ISO8601}}{dim,cyan}
%style{Thread}{dim,blue}%style{[}{red}%style{%T}{dim,cyan}%style{]}{red}
%style{[}{red}%style{%-8c{2}}{dim,blue}%style{]}{red}
%highlight{%-5p}%style{:}{red} %m%n
logger.shellaudit.name = org.apache.accumulo.shell.Shell.audit
logger.shellaudit.level = warn
@@ -43,4 +43,3 @@ logger.zookeeper.level = error
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDERR
-
diff --git a/assemble/pom.xml b/assemble/pom.xml
index aa7db534f1..ecd9db9444 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo</artifactId>
<packaging>pom</packaging>
diff --git a/core/pom.xml b/core/pom.xml
index d0efea0cb0..02ecf146ba 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-core</artifactId>
<name>Apache Accumulo Core</name>
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index fbe17c6386..ada56571b1 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -86,10 +86,12 @@ public class ZooKeeperInstance implements Instance {
* @param zooKeepers A comma separated list of zoo keeper server locations.
Each location can
* contain an optional port, of the format host:port.
*/
+ @Deprecated
public ZooKeeperInstance(String instanceName, String zooKeepers) {
this(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zooKeepers));
}
+ @Deprecated
ZooKeeperInstance(ClientConfiguration config, ZooCacheFactory zcf) {
checkArgument(config != null, "config is null");
// Enable singletons before before getting a zoocache
@@ -118,10 +120,12 @@ public class ZooKeeperInstance implements Instance {
* specific to Accumulo.
* @since 1.9.0
*/
+ @Deprecated
public ZooKeeperInstance(ClientConfiguration config) {
this(config, new ZooCacheFactory());
}
+ @Deprecated
@Override
public String getInstanceID() {
if (instanceId == null) {
@@ -131,18 +135,19 @@ public class ZooKeeperInstance implements Instance {
return instanceId;
}
+ @Deprecated
@Override
public List<String> getMasterLocations() {
return ClientContext.getManagerLocations(zooCache, getInstanceID());
}
+ @Deprecated
@Override
public String getRootTabletLocation() {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Looking up root tablet location in zookeeper.",
- Thread.currentThread().getId());
+ log.trace("Looking up root tablet location in zookeeper.");
timer = Timer.startNew();
}
@@ -150,7 +155,7 @@ public class ZooKeeperInstance implements Instance {
.getRootMetadata(Constants.ZROOT + "/" + getInstanceID(),
zooCache).getLocation();
if (timer != null) {
- log.trace("tid={} Found root tablet at {} in {}",
Thread.currentThread().getId(), loc,
+ log.trace("Found root tablet at {} in {}", loc,
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
@@ -161,6 +166,7 @@ public class ZooKeeperInstance implements Instance {
return loc.getHostPort();
}
+ @Deprecated
@Override
public String getInstanceName() {
if (instanceName == null) {
@@ -171,16 +177,19 @@ public class ZooKeeperInstance implements Instance {
return instanceName;
}
+ @Deprecated
@Override
public String getZooKeepers() {
return zooKeepers;
}
+ @Deprecated
@Override
public int getZooKeepersSessionTimeOut() {
return zooKeepersSessionTimeOut;
}
+ @Deprecated
@Override
public Connector getConnector(String principal, AuthenticationToken token)
throws AccumuloException, AccumuloSecurityException {
@@ -193,6 +202,7 @@ public class ZooKeeperInstance implements Instance {
new ClientContext(SingletonReservation.noop(), info, serverConf,
null));
}
+ @Deprecated
@Override
public String toString() {
StringBuilder sb = new StringBuilder(64);
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
index 7dc37c3545..d267d4b7d4 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
@@ -486,8 +486,7 @@ public class ClientContext implements AccumuloClient {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Looking up root tablet location in zookeeper.",
- Thread.currentThread().getId());
+ log.trace("Looking up root tablet location in zookeeper.");
timer = Timer.startNew();
}
@@ -495,7 +494,7 @@ public class ClientContext implements AccumuloClient {
getAmple().readTablet(RootTable.EXTENT, ReadConsistency.EVENTUAL,
LOCATION).getLocation();
if (timer != null) {
- log.trace("tid={} Found root tablet at {} in {}",
Thread.currentThread().getId(), loc,
+ log.trace("Found root tablet at {} in {}", loc,
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
@@ -524,15 +523,14 @@ public class ClientContext implements AccumuloClient {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Looking up manager location in zookeeper.",
Thread.currentThread().getId());
+ log.trace("Looking up manager location in zookeeper.");
timer = Timer.startNew();
}
byte[] loc = zooCache.getLockData(zLockManagerPath);
if (timer != null) {
- log.trace("tid={} Found manager at {} in {}",
Thread.currentThread().getId(),
- (loc == null ? "null" : new String(loc, UTF_8)),
+ log.trace("Found manager at {} in {}", (loc == null ? "null" : new
String(loc, UTF_8)),
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
index 451496ed9c..392ba64bad 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
@@ -85,15 +85,15 @@ public class NamespaceOperationsImpl extends
NamespaceOperationsHelper {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Fetching list of namespaces...",
Thread.currentThread().getId());
+ log.trace("Fetching list of namespaces...");
timer = Timer.startNew();
}
TreeSet<String> namespaces = new
TreeSet<>(Namespaces.getNameToIdMap(context).keySet());
if (timer != null) {
- log.trace("tid={} Fetched {} namespaces in {}",
Thread.currentThread().getId(),
- namespaces.size(), String.format("%.3f secs",
timer.elapsed(MILLISECONDS) / 1000.0));
+ log.trace("Fetched {} namespaces in {}", namespaces.size(),
+ String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
return namespaces;
@@ -106,15 +106,14 @@ public class NamespaceOperationsImpl extends
NamespaceOperationsHelper {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Checking if namespace {} exists",
Thread.currentThread().getId(),
- namespace);
+ log.trace("Checking if namespace {} exists", namespace);
timer = Timer.startNew();
}
boolean exists = Namespaces.namespaceNameExists(context, namespace);
if (timer != null) {
- log.trace("tid={} Checked existence of {} in {}",
Thread.currentThread().getId(), exists,
+ log.trace("Checked existence of {} in {}", exists,
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
index 7fa49e5794..bfe0b8c56e 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
@@ -102,8 +102,7 @@ public class RootTabletLocator extends TabletLocator {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Looking up root tablet location in zookeeper.",
- Thread.currentThread().getId());
+ log.trace("Looking up root tablet location in zookeeper.");
timer = Timer.startNew();
}
@@ -111,7 +110,7 @@ public class RootTabletLocator extends TabletLocator {
.readTablet(RootTable.EXTENT, ReadConsistency.EVENTUAL,
LOCATION).getLocation();
if (timer != null) {
- log.trace("tid={} Found root tablet at {} in {}",
Thread.currentThread().getId(), loc,
+ log.trace("Found root tablet at {} in {}", loc,
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 0fc1e52ac3..38732fddf7 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -180,15 +180,15 @@ public class TableOperationsImpl extends
TableOperationsHelper {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Fetching list of tables...",
Thread.currentThread().getId());
+ log.trace("Fetching list of tables...");
timer = Timer.startNew();
}
TreeSet<String> tableNames = new
TreeSet<>(context.getTableNameToIdMap().keySet());
if (timer != null) {
- log.trace("tid={} Fetched {} table names in {}",
Thread.currentThread().getId(),
- tableNames.size(), String.format("%.3f secs",
timer.elapsed(MILLISECONDS) / 1000.0));
+ log.trace("Fetched {} table names in {}", tableNames.size(),
+ String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
return tableNames;
@@ -205,14 +205,14 @@ public class TableOperationsImpl extends
TableOperationsHelper {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Checking if table {} exists...",
Thread.currentThread().getId(), tableName);
+ log.trace("Checking if table {} exists...", tableName);
timer = Timer.startNew();
}
boolean exists = context.getTableNameToIdMap().containsKey(tableName);
if (timer != null) {
- log.trace("tid={} Checked existence of {} in {}",
Thread.currentThread().getId(), exists,
+ log.trace("Checked existence of {} in {}", exists,
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
@@ -571,8 +571,7 @@ public class TableOperationsImpl extends
TableOperationsHelper {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Splitting tablet {} on {} at {}",
Thread.currentThread().getId(),
- tl.tablet_extent, address, split);
+ log.trace("Splitting tablet {} on {} at {}", tl.tablet_extent,
address, split);
timer = Timer.startNew();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
index d46f7c9ef4..7bc8d3f4ed 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
@@ -170,8 +170,7 @@ public class TabletLocatorImpl extends TabletLocator {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Binning {} mutations for table {}",
Thread.currentThread().getId(),
- mutations.size(), tableId);
+ log.trace("Binning {} mutations for table {}", mutations.size(),
tableId);
timer = Timer.startNew();
}
@@ -233,9 +232,8 @@ public class TabletLocatorImpl extends TabletLocator {
}
if (timer != null) {
- log.trace("tid={} Binned {} mutations for table {} to {} tservers in {}",
- Thread.currentThread().getId(), mutations.size(), tableId,
binnedMutations.size(),
- String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
+ log.trace("Binned {} mutations for table {} to {} tservers in {}",
mutations.size(), tableId,
+ binnedMutations.size(), String.format("%.3f secs",
timer.elapsed(MILLISECONDS) / 1000.0));
}
}
@@ -375,8 +373,7 @@ public class TabletLocatorImpl extends TabletLocator {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Binning {} ranges for table {}",
Thread.currentThread().getId(),
- ranges.size(), tableId);
+ log.trace("Binning {} ranges for table {}", ranges.size(), tableId);
timer = Timer.startNew();
}
@@ -411,9 +408,8 @@ public class TabletLocatorImpl extends TabletLocator {
}
if (timer != null) {
- log.trace("tid={} Binned {} ranges for table {} to {} tservers in {}",
- Thread.currentThread().getId(), ranges.size(), tableId,
binnedRanges.size(),
- String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
+ log.trace("Binned {} ranges for table {} to {} tservers in {}",
ranges.size(), tableId,
+ binnedRanges.size(), String.format("%.3f secs",
timer.elapsed(MILLISECONDS) / 1000.0));
}
return failures;
@@ -484,8 +480,8 @@ public class TabletLocatorImpl extends TabletLocator {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Locating tablet table={} row={} skipRow={} retry={}",
- Thread.currentThread().getId(), tableId, TextUtil.truncate(row),
skipRow, retry);
+ log.trace("Locating tablet table={} row={} skipRow={} retry={}",
tableId,
+ TextUtil.truncate(row), skipRow, retry);
timer = Timer.startNew();
}
@@ -504,8 +500,8 @@ public class TabletLocatorImpl extends TabletLocator {
}
if (timer != null) {
- log.trace("tid={} Located tablet {} at {} in {}",
Thread.currentThread().getId(),
- (tl == null ? "null" : tl.tablet_extent), (tl == null ? "null" :
tl.tablet_location),
+ log.trace("Located tablet {} at {} in {}", (tl == null ? "null" :
tl.tablet_extent),
+ (tl == null ? "null" : tl.tablet_location),
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
index 6890171517..d05c483335 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
@@ -820,11 +820,9 @@ public class TabletServerBatchReaderIterator implements
Iterator<Entry<Key,Value
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace(
- "tid={} Starting multi scan, tserver={} #tablets={} #ranges={}
ssil={} ssio={}",
- Thread.currentThread().getId(), server, requested.size(),
- sumSizes(requested.values()), options.serverSideIteratorList,
- options.serverSideIteratorOptions);
+ log.trace("Starting multi scan, tserver={} #tablets={} #ranges={}
ssil={} ssio={}",
+ server, requested.size(), sumSizes(requested.values()),
+ options.serverSideIteratorList,
options.serverSideIteratorOptions);
timer = Timer.startNew();
}
@@ -856,8 +854,7 @@ public class TabletServerBatchReaderIterator implements
Iterator<Entry<Key,Value
MultiScanResult scanResult = imsr.result;
if (timer != null) {
- log.trace("tid={} Got 1st multi scan results, #results={} {} in {}",
- Thread.currentThread().getId(), scanResult.results.size(),
+ log.trace("Got 1st multi scan results, #results={} {} in {}",
scanResult.results.size(),
(scanResult.more ? "scanID=" + imsr.scanID : ""),
String.format("%.3f secs", timer.elapsed(MILLISECONDS) /
1000.0));
}
@@ -884,17 +881,16 @@ public class TabletServerBatchReaderIterator implements
Iterator<Entry<Key,Value
timeoutTracker.check();
if (timer != null) {
- log.trace("tid={} oid={} Continuing multi scan, scanid={}",
- Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
+ log.trace("oid={} Continuing multi scan, scanid={}",
nextOpid.get(), imsr.scanID);
timer.restart();
}
scanResult = client.continueMultiScan(TraceUtil.traceInfo(),
imsr.scanID, busyTimeout);
if (timer != null) {
- log.trace("tid={} oid={} Got more multi scan results, #results={}
{} in {}",
- Thread.currentThread().getId(), nextOpid.getAndIncrement(),
- scanResult.results.size(), (scanResult.more ? " scanID=" +
imsr.scanID : ""),
+ log.trace("oid={} Got more multi scan results, #results={} {} in
{}",
+ nextOpid.getAndIncrement(), scanResult.results.size(),
+ (scanResult.more ? " scanID=" + imsr.scanID : ""),
String.format("%.3f secs", timer.elapsed(MILLISECONDS) /
1000.0));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
index 62f8a271a2..43ba332221 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
@@ -644,13 +644,12 @@ public class TabletServerBatchWriter implements
AutoCloseable {
if (rf != null) {
if (log.isTraceEnabled()) {
- log.trace("tid={} Requeuing {} failed mutations",
Thread.currentThread().getId(),
- rf.size());
+ log.trace("Requeuing {} failed mutations", rf.size());
}
addFailedMutations(rf);
}
- } catch (Exception t) {
- updateUnknownErrors("tid=" + Thread.currentThread().getId()
+ } catch (RuntimeException t) {
+ updateUnknownErrors(Threads.toString(Thread.currentThread())
+ " Failed to requeue failed mutations " + t.getMessage(), t);
executor.remove(task);
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
index 0447e5d423..59871e9917 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
@@ -694,7 +694,7 @@ public class ThriftScanner {
+ loc.tablet_extent + " range=" + scanState.range + " ssil="
+ scanState.serverSideIteratorList + " ssio=" +
scanState.serverSideIteratorOptions
+ " context=" + scanState.classLoaderContext;
- log.trace("tid={} {}", Thread.currentThread().getId(), msg);
+ log.trace("{}", msg);
timer = Timer.startNew();
}
@@ -728,7 +728,7 @@ public class ThriftScanner {
Thread.currentThread().setName(msg);
if (log.isTraceEnabled()) {
- log.trace("tid={} {}", Thread.currentThread().getId(), msg);
+ log.trace("{}", msg);
timer = Timer.startNew();
}
@@ -741,8 +741,7 @@ public class ThriftScanner {
if (sr.more) {
if (timer != null) {
- log.trace("tid={} Finished scan in {} #results={} scanid={}",
- Thread.currentThread().getId(),
+ log.trace("Finished scan in {} #results={} scanid={}",
String.format("%.3f secs", timer.elapsed(MILLISECONDS) /
1000.0), sr.results.size(),
scanState.scanID);
}
@@ -753,8 +752,7 @@ public class ThriftScanner {
scanState.finished = true;
if (timer != null) {
- log.trace("tid={} Completely finished scan in {} #results={}",
- Thread.currentThread().getId(),
+ log.trace("Completely finished scan in {} #results={}",
String.format("%.3f secs", timer.elapsed(MILLISECONDS) /
1000.0),
sr.results.size());
}
@@ -765,16 +763,14 @@ public class ThriftScanner {
scanState.skipStartRow = true;
if (timer != null) {
- log.trace("tid={} Finished scanning tablet in {} #results={}",
- Thread.currentThread().getId(),
+ log.trace("Finished scanning tablet in {} #results={}",
String.format("%.3f secs", timer.elapsed(MILLISECONDS) /
1000.0),
sr.results.size());
}
} else {
scanState.finished = true;
if (timer != null) {
- log.trace("tid={} Completely finished in {} #results={}",
- Thread.currentThread().getId(),
+ log.trace("Completely finished in {} #results={}",
String.format("%.3f secs", timer.elapsed(MILLISECONDS) /
1000.0),
sr.results.size());
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
index ce198a7e85..023ec1b844 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
@@ -87,9 +87,8 @@ public class MetadataLocationObtainer implements
TabletLocationObtainer {
Timer timer = null;
if (log.isTraceEnabled()) {
- log.trace("tid={} Looking up in {} row={} extent={} tserver={}",
- Thread.currentThread().getId(), src.tablet_extent.tableId(),
TextUtil.truncate(row),
- src.tablet_extent, src.tablet_location);
+ log.trace("Looking up in {} row={} extent={} tserver={}",
src.tablet_extent.tableId(),
+ TextUtil.truncate(row), src.tablet_extent, src.tablet_location);
timer = Timer.startNew();
}
@@ -123,8 +122,7 @@ public class MetadataLocationObtainer implements
TabletLocationObtainer {
}
if (timer != null) {
- log.trace("tid={} Got {} results from {} in {}",
Thread.currentThread().getId(),
- results.size(), src.tablet_extent,
+ log.trace("Got {} results from {} in {}", results.size(),
src.tablet_extent,
String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/threads/Threads.java
b/core/src/main/java/org/apache/accumulo/core/util/threads/Threads.java
index 611f7a6755..71a53da507 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/threads/Threads.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/threads/Threads.java
@@ -83,4 +83,19 @@ public class Threads {
return createNonCriticalThread(name, priority, wrapped);
}
+
+ // only Java 19 and later have the thread ID in the toString; this can be
removed when the
+ // required Java version is at least 19
+ public static String toString(Thread t) {
+ StringBuilder sb = new StringBuilder("Thread[#");
+
sb.append(t.getId()).append(",").append(t.getName()).append(",").append(t.getPriority())
+ .append(",");
+ ThreadGroup group = t.getThreadGroup();
+ if (group != null) {
+ sb.append(group.getName());
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
b/core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
index c97a7d231e..fdf122de98 100644
---
a/core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
+++
b/core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
@@ -18,7 +18,9 @@
*/
package org.apache.accumulo.core.classloader;
+import java.io.UncheckedIOException;
import java.net.MalformedURLException;
+import java.net.URI;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.stream.Stream;
@@ -35,9 +37,9 @@ public class URLClassLoaderFactory implements
ContextClassLoaderFactory {
// The context name is the classpath.
URL[] urls = Stream.of(contextName.split(COMMA)).map(p -> {
try {
- return new URL(p);
+ return URI.create(p).toURL();
} catch (MalformedURLException e) {
- throw new IllegalArgumentException("Error creating URL from classpath
segment: " + p, e);
+ throw new UncheckedIOException(e);
}
}).toArray(URL[]::new);
return URLClassLoader.newInstance(urls);
diff --git a/core/src/test/resources/log4j2-test.properties
b/core/src/test/resources/log4j2-test.properties
index 46192dd2a0..2d20aeb263 100644
--- a/core/src/test/resources/log4j2-test.properties
+++ b/core/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloCoreTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%t] %-5p %c %x - %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core.iterators.system.VisibilityFilter
logger.01.level = fatal
@@ -58,4 +58,3 @@ logger.10.level = fatal
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/hadoop-mapreduce/pom.xml b/hadoop-mapreduce/pom.xml
index 31664e9bb4..a690773cfe 100644
--- a/hadoop-mapreduce/pom.xml
+++ b/hadoop-mapreduce/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-hadoop-mapreduce</artifactId>
<name>Apache Accumulo Hadoop MapReduce</name>
diff --git a/hadoop-mapreduce/src/test/resources/log4j2-test.properties
b/hadoop-mapreduce/src/test/resources/log4j2-test.properties
index 29deee2124..2d20aeb263 100644
--- a/hadoop-mapreduce/src/test/resources/log4j2-test.properties
+++ b/hadoop-mapreduce/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloHadoopMapReduceTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%t] %-5p %c %x - %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core.iterators.system.VisibilityFilter
logger.01.level = fatal
@@ -58,4 +58,3 @@ logger.10.level = fatal
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/iterator-test-harness/pom.xml b/iterator-test-harness/pom.xml
index 8f09a5836a..8a0f881fa0 100644
--- a/iterator-test-harness/pom.xml
+++ b/iterator-test-harness/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-iterator-test-harness</artifactId>
<name>Apache Accumulo Iterator Test Harness</name>
diff --git a/iterator-test-harness/src/test/resources/log4j2-test.properties
b/iterator-test-harness/src/test/resources/log4j2-test.properties
index 5bc25f94d4..c6b08975eb 100644
--- a/iterator-test-harness/src/test/resources/log4j2-test.properties
+++ b/iterator-test-harness/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloIteratorTestHarnessTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core.clientImpl.ServerClient
logger.01.level = error
@@ -40,4 +40,3 @@ logger.04.level = debug
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/minicluster/pom.xml b/minicluster/pom.xml
index bcec0beb56..d49cac87e5 100644
--- a/minicluster/pom.xml
+++ b/minicluster/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-minicluster</artifactId>
<name>Apache Accumulo MiniCluster</name>
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
index 94b5147ad9..aa1b9f3e26 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
@@ -93,7 +93,6 @@ import org.apache.accumulo.server.util.AccumuloStatus;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.accumulo.server.util.ZooZap;
import org.apache.accumulo.start.Main;
-import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -200,7 +199,6 @@ public class MiniAccumuloClusterImpl implements
AccumuloCluster {
conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, "1");
conf.set("dfs.support.append", "true");
conf.set("dfs.datanode.synconclose", "true");
- conf.set("dfs.datanode.data.dir.perm",
MiniDFSUtil.computeDatanodeDirectoryPermission());
config.getHadoopConfOverrides().forEach((k, v) -> conf.set(k, v));
String oldTestBuildData = System.setProperty("test.build.data",
dfs.getAbsolutePath());
miniDFS.set(new
MiniDFSCluster.Builder(conf).numDataNodes(config.getNumDataNodes()).build());
diff --git a/minicluster/src/test/resources/log4j2-test.properties
b/minicluster/src/test/resources/log4j2-test.properties
index 125d2d7c1f..0b7617cca1 100644
--- a/minicluster/src/test/resources/log4j2-test.properties
+++ b/minicluster/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloMiniclusterTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core.clientImpl.ServerClient
logger.01.level = error
@@ -34,4 +34,3 @@ logger.02.level = error
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/pom.xml b/pom.xml
index 462f665396..be29dba440 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>37</version>
+ <relativePath />
</parent>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
diff --git a/server/base/src/test/resources/log4j2-test.properties
b/server/base/src/test/resources/log4j2-test.properties
index 3108652386..52a2551c8b 100644
--- a/server/base/src/test/resources/log4j2-test.properties
+++ b/server/base/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloServerBaseTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.server.util.TabletIterator
logger.01.level = error
diff --git
a/server/compaction-coordinator/src/test/resources/log4j2-test.properties
b/server/compaction-coordinator/src/test/resources/log4j2-test.properties
index b224053223..59f2f78fec 100644
--- a/server/compaction-coordinator/src/test/resources/log4j2-test.properties
+++ b/server/compaction-coordinator/src/test/resources/log4j2-test.properties
@@ -18,17 +18,13 @@
#
dest = err
-name = AccumuloCoreTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%t] %-5p %c %x - %m%n
-
-#logger.01.name = org.apache.accumulo.core.iterators.system.VisibilityFilter
-#logger.01.level = fatal
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
rootLogger.level = debug
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/server/compactor/src/test/resources/log4j2-test.properties
b/server/compactor/src/test/resources/log4j2-test.properties
index fc0ce5ede0..2c64203f6d 100644
--- a/server/compactor/src/test/resources/log4j2-test.properties
+++ b/server/compactor/src/test/resources/log4j2-test.properties
@@ -18,17 +18,13 @@
#
dest = err
-name = AccumuloCoreTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%t] %-5p %c %x - %m%n
-
-#logger.01.name = org.apache.accumulo.core.iterators.system.VisibilityFilter
-#logger.01.level = fatal
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/server/gc/src/test/resources/log4j2-test.properties
b/server/gc/src/test/resources/log4j2-test.properties
index 06203a2c96..8a630d025b 100644
--- a/server/gc/src/test/resources/log4j2-test.properties
+++ b/server/gc/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloGcTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.hadoop.util.NativeCodeLoader
logger.01.level = error
@@ -34,4 +34,3 @@ logger.02.level = warn
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/server/manager/src/test/resources/log4j2-test.properties
b/server/manager/src/test/resources/log4j2-test.properties
index 29c6d66a28..451000c878 100644
--- a/server/manager/src/test/resources/log4j2-test.properties
+++ b/server/manager/src/test/resources/log4j2-test.properties
@@ -18,17 +18,16 @@
#
dest = err
-name = AccumuloManagerTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.hadoop.util.NativeCodeLoader
logger.01.level = error
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 110f032d94..122d8c927d 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -24,7 +24,7 @@ import static java.util.concurrent.TimeUnit.MINUTES;
import static
org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
import java.net.InetAddress;
-import java.net.URL;
+import java.net.URI;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -524,14 +524,14 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
rootContext = rootContext + "/";
}
try {
- URL url = new URL(server.isSecure() ? "https" : "http",
monitorHostAndPort.getHost(),
- server.getPort(), rootContext);
+ var uri = new URI(server.isSecure() ? "https" : "http", null,
monitorHostAndPort.getHost(),
+ server.getPort(), rootContext, null, null);
final String path = context.getZooKeeperRoot() +
Constants.ZMONITOR_HTTP_ADDR;
final ZooReaderWriter zoo = context.getZooReaderWriter();
// Delete before we try to re-create in case the previous session hasn't
yet expired
zoo.delete(path);
- zoo.putEphemeralData(path, url.toString().getBytes(UTF_8));
- log.info("Set monitor address in zookeeper to {}", url);
+ zoo.putEphemeralData(path, uri.toString().getBytes(UTF_8));
+ log.info("Set monitor address in zookeeper to {}", uri);
} catch (Exception ex) {
log.error("Unable to advertise monitor HTTP address in zookeeper", ex);
}
diff --git a/server/monitor/src/test/resources/log4j2-test.properties
b/server/monitor/src/test/resources/log4j2-test.properties
index 9901a56a0d..2c64203f6d 100644
--- a/server/monitor/src/test/resources/log4j2-test.properties
+++ b/server/monitor/src/test/resources/log4j2-test.properties
@@ -18,14 +18,13 @@
#
dest = err
-name = AccumuloMonitorTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/server/tserver/src/test/resources/log4j2-test.properties
b/server/tserver/src/test/resources/log4j2-test.properties
index 041d5d6149..024072329d 100644
--- a/server/tserver/src/test/resources/log4j2-test.properties
+++ b/server/tserver/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloTserverTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} [%c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core
logger.01.level = debug
@@ -76,4 +76,3 @@ logger.16.level = info
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/shell/pom.xml b/shell/pom.xml
index 787959690d..09a72616ab 100644
--- a/shell/pom.xml
+++ b/shell/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-shell</artifactId>
<name>Apache Accumulo Shell</name>
diff --git a/shell/src/test/resources/log4j2-test.properties
b/shell/src/test/resources/log4j2-test.properties
index 5fc36ac912..eede6872c0 100644
--- a/shell/src/test/resources/log4j2-test.properties
+++ b/shell/src/test/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloShellTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%t] %-5p %c %x - %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core.iterators.system.VisibilityFilter
logger.01.level = fatal
@@ -52,4 +52,3 @@ logger.08.level = fatal
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/start/pom.xml b/start/pom.xml
index e0407189cf..ac1164293c 100644
--- a/start/pom.xml
+++ b/start/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-start</artifactId>
<name>Apache Accumulo Start</name>
diff --git
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/MiniDFSUtil.java
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/MiniDFSUtil.java
deleted file mode 100644
index 59288a5e46..0000000000
---
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/MiniDFSUtil.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.start.classloader.vfs;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-public class MiniDFSUtil {
-
- public static String computeDatanodeDirectoryPermission() {
- // MiniDFSCluster will check the permissions on the data directories, but
does not
- // do a good job of setting them properly. We need to get the users umask
and set
- // the appropriate Hadoop property so that the data directories will be
created
- // with the correct permissions.
- try {
- Process p = Runtime.getRuntime().exec("/bin/sh -c umask");
- try (BufferedReader bri = new BufferedReader(new
InputStreamReader(p.getInputStream()))) {
- String line = bri.readLine();
- p.waitFor();
-
- if (line == null) {
- throw new IOException("umask input stream closed prematurely");
- }
- short umask = Short.parseShort(line.trim(), 8);
- // Need to set permission to 777 xor umask
- // leading zero makes java interpret as base 8
- int newPermission = 0777 ^ umask;
-
- return String.format("%03o", newPermission);
- }
- } catch (Exception e) {
- throw new RuntimeException("Error getting umask from O/S", e);
- }
- }
-
-}
diff --git
a/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
b/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
index dd0b444e32..141f8847f2 100644
--- a/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
+++ b/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
-import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.cache.DefaultFilesCache;
@@ -64,9 +63,6 @@ public class AccumuloDFSBase {
// Setup HDFS
conf = new Configuration();
- conf.set("hadoop.security.token.service.use_ip", "true");
-
- conf.set("dfs.datanode.data.dir.perm",
MiniDFSUtil.computeDatanodeDirectoryPermission());
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M
blocksize
try {
diff --git a/start/src/test/resources/log4j2-test.properties
b/start/src/test/resources/log4j2-test.properties
index 0ec9c0cd23..3c1318b12b 100644
--- a/start/src/test/resources/log4j2-test.properties
+++ b/start/src/test/resources/log4j2-test.properties
@@ -18,17 +18,16 @@
#
dest = err
-name = AccumuloStartTestLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%t} %-5p %c %x - %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.commons.vfs2.impl.DefaultFileSystemManager
logger.01.level = error
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git a/test/pom.xml b/test/pom.xml
index 27bdfd8dfc..c63128737c 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -25,6 +25,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>2.1.5-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
<artifactId>accumulo-test</artifactId>
<name>Apache Accumulo Testing</name>
diff --git
a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
index 2480641b69..c925e089c5 100644
--- a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
@@ -21,7 +21,7 @@ package org.apache.accumulo.test;
import static
org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.net.URL;
+import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -75,7 +75,7 @@ public class CountNameNodeOpsBulkIT extends
ConfigurableMacBase {
justification = "path provided by test; url provided by test")
private Map<?,?> getStats() throws Exception {
String uri = getCluster().getMiniDfs().getHttpUri(0);
- URL url = new URL(uri + "/jmx");
+ var url = new URI(uri + "/jmx").toURL();
log.debug("Fetching web page " + url);
String jsonString = FunctionalTestUtils.readWebPage(url).body();
Map<?,?> jsonObject = new Gson().fromJson(jsonString, Map.class);
diff --git
a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java
b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java
index 33588a882b..acdf93e329 100644
---
a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java
@@ -23,7 +23,7 @@ import static
org.apache.accumulo.harness.AccumuloITBase.MINI_CLUSTER_ONLY;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.Socket;
-import java.net.URL;
+import java.net.URI;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -94,7 +94,7 @@ public class ThriftServerBindsBeforeZooKeeperLockIT extends
AccumuloClusterHarne
monitor = startProcess(cluster, ServerType.MONITOR, freePort);
while (true) {
- URL url = new URL(monitorUrl);
+ var url = new URI(monitorUrl).toURL();
try {
HttpURLConnection cnxn = (HttpURLConnection) url.openConnection();
final int responseCode = cnxn.getResponseCode();
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MonitorSslIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MonitorSslIT.java
index a588cb12a4..aa7617c3c3 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MonitorSslIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MonitorSslIT.java
@@ -21,7 +21,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
-import java.net.URL;
+import java.net.URI;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
@@ -142,7 +142,7 @@ public class MonitorSslIT extends ConfigurableMacBase {
}
}
}
- URL url = new URL(monitorLocation);
+ var url = new URI(monitorLocation).toURL();
log.debug("Fetching web page {}", url);
String result = FunctionalTestUtils.readWebPage(url).body();
assertTrue(result.length() > 100);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index c4cb9574a9..265f0832b1 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@ -28,7 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
-import java.net.URL;
+import java.net.URI;
import java.security.cert.X509Certificate;
import java.time.Duration;
import java.util.ArrayList;
@@ -159,7 +159,7 @@ public class ReadWriteIT extends AccumuloClusterHarness {
HttpsURLConnection.setDefaultHostnameVerifier(new
TestHostnameVerifier());
}
}
- URL url = new URL(monitorLocation);
+ var url = new URI(monitorLocation).toURL();
log.debug("Fetching web page {}", url);
String result = FunctionalTestUtils.readWebPage(url).body();
assertTrue(result.length() > 100);
diff --git
a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
deleted file mode 100644
index ae789e0d88..0000000000
---
a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++ /dev/null
@@ -1,615 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.test.performance.scan;
-
-import static org.apache.accumulo.harness.AccumuloITBase.random;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.clientImpl.ClientContext;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.data.ArrayByteSequence;
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Column;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
-import org.apache.accumulo.core.file.FileOperations;
-import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil;
-import
org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator;
-import org.apache.accumulo.core.iteratorsImpl.system.ColumnQualifierFilter;
-import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator;
-import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator.Behavior;
-import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
-import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator;
-import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter;
-import org.apache.accumulo.core.metadata.MetadataServicer;
-import org.apache.accumulo.core.metadata.TabletFile;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.core.util.Stat;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.iterators.SystemIteratorEnvironmentImpl;
-import org.apache.accumulo.server.util.MetadataTableUtil;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
-public class CollectTabletStats {
-
- private static final Logger log =
LoggerFactory.getLogger(CollectTabletStats.class);
-
- static class CollectOptions extends ServerUtilOpts {
- @Parameter(names = {"-t", "--table"}, required = true, description =
"table to use")
- String tableName;
- @Parameter(names = "--iterations", description = "number of iterations")
- int iterations = 3;
- @Parameter(names = "--numThreads", description = "number of threads")
- int numThreads = 1;
- @Parameter(names = "-f", description = "select far tablets, default is to
use local tablets")
- boolean selectFarTablets = false;
- @Parameter(names = "--columns", description = "comma separated list of
columns")
- String columns;
- }
-
- public static void main(String[] args) throws Exception {
-
- final CollectOptions opts = new CollectOptions();
- opts.parseArgs(CollectTabletStats.class.getName(), args);
-
- String[] columnsTmp = {};
- if (opts.columns != null) {
- columnsTmp = opts.columns.split(",");
- }
- final String[] columns = columnsTmp;
-
- ServerContext context = opts.getServerContext();
- final VolumeManager fs = context.getVolumeManager();
-
- TableId tableId = context.getTableId(opts.tableName);
- if (tableId == null) {
- log.error("Unable to find table named {}", opts.tableName);
- System.exit(-1);
- }
-
- TreeMap<KeyExtent,String> tabletLocations = new TreeMap<>();
- List<KeyExtent> candidates =
- findTablets(context, !opts.selectFarTablets, opts.tableName,
tabletLocations);
-
- if (candidates.size() < opts.numThreads) {
- System.err.println("ERROR : Unable to find " + opts.numThreads + " "
- + (opts.selectFarTablets ? "far" : "local") + " tablets");
- System.exit(-1);
- }
-
- List<KeyExtent> tabletsToTest = selectRandomTablets(opts.numThreads,
candidates);
-
- Map<KeyExtent,List<TabletFile>> tabletFiles = new HashMap<>();
-
- for (KeyExtent ke : tabletsToTest) {
- List<TabletFile> files = getTabletFiles(context, ke);
- tabletFiles.put(ke, files);
- }
-
- System.out.println();
- System.out.println("run location : " +
InetAddress.getLocalHost().getHostName() + "/"
- + InetAddress.getLocalHost().getHostAddress());
- System.out.println("num threads : " + opts.numThreads);
- System.out.println("table : " + opts.tableName);
- System.out.println("table id : " + tableId);
-
- for (KeyExtent ke : tabletsToTest) {
- System.out.println("\t *** Information about tablet " + ke.getUUID() + "
*** ");
- System.out.println("\t\t# files in tablet : " +
tabletFiles.get(ke).size());
- System.out.println("\t\ttablet location : " + tabletLocations.get(ke));
- reportHdfsBlockLocations(context, tabletFiles.get(ke));
- }
-
- System.out.println("%n*** RUNNING TEST ***%n");
-
- ExecutorService threadPool = Executors.newFixedThreadPool(opts.numThreads);
-
- for (int i = 0; i < opts.iterations; i++) {
-
- ArrayList<Test> tests = new ArrayList<>();
-
- for (final KeyExtent ke : tabletsToTest) {
- final List<TabletFile> files = tabletFiles.get(ke);
- Test test = new Test(ke) {
- @Override
- public int runTest() throws Exception {
- return readFiles(fs, context.getConfiguration(), files, ke,
columns);
- }
-
- };
-
- tests.add(test);
- }
-
- runTest("read files", tests, opts.numThreads, threadPool);
- }
-
- for (int i = 0; i < opts.iterations; i++) {
-
- ArrayList<Test> tests = new ArrayList<>();
-
- for (final KeyExtent ke : tabletsToTest) {
- final List<TabletFile> files = tabletFiles.get(ke);
- Test test = new Test(ke) {
- @Override
- public int runTest() throws Exception {
- return readFilesUsingIterStack(fs, context, files, opts.auths, ke,
columns, false);
- }
- };
-
- tests.add(test);
- }
-
- runTest("read tablet files w/ system iter stack", tests,
opts.numThreads, threadPool);
- }
-
- for (int i = 0; i < opts.iterations; i++) {
- ArrayList<Test> tests = new ArrayList<>();
-
- for (final KeyExtent ke : tabletsToTest) {
- final List<TabletFile> files = tabletFiles.get(ke);
- Test test = new Test(ke) {
- @Override
- public int runTest() throws Exception {
- return readFilesUsingIterStack(fs, context, files, opts.auths, ke,
columns, true);
- }
- };
-
- tests.add(test);
- }
-
- runTest("read tablet files w/ table iter stack", tests, opts.numThreads,
threadPool);
- }
-
- try (AccumuloClient client =
Accumulo.newClient().from(opts.getClientProps()).build()) {
- for (int i = 0; i < opts.iterations; i++) {
- ArrayList<Test> tests = new ArrayList<>();
- for (final KeyExtent ke : tabletsToTest) {
- Test test = new Test(ke) {
- @Override
- public int runTest() throws Exception {
- return scanTablet(client, opts.tableName, opts.auths,
ke.prevEndRow(), ke.endRow(),
- columns);
- }
- };
- tests.add(test);
- }
- runTest("read tablet data through accumulo", tests, opts.numThreads,
threadPool);
- }
-
- for (final KeyExtent ke : tabletsToTest) {
- threadPool.execute(() -> {
- try {
- calcTabletStats(client, opts.tableName, opts.auths, ke, columns);
- } catch (Exception e) {
- log.error("Failed to calculate tablet stats.", e);
- }
- });
- }
- }
-
- threadPool.shutdown();
- }
-
- private abstract static class Test implements Runnable {
-
- private int count;
- private long t1;
- private long t2;
- private CountDownLatch startCdl, finishCdl;
- private KeyExtent ke;
-
- Test(KeyExtent ke) {
- this.ke = ke;
- }
-
- public abstract int runTest() throws Exception;
-
- void setSignals(CountDownLatch scdl, CountDownLatch fcdl) {
- this.startCdl = scdl;
- this.finishCdl = fcdl;
- }
-
- @Override
- public void run() {
-
- try {
- startCdl.await();
- } catch (InterruptedException e) {
- log.error("startCdl.await() failed.", e);
- }
-
- t1 = System.currentTimeMillis();
-
- try {
- count = runTest();
- } catch (Exception e) {
- log.error("runTest() failed.", e);
- }
-
- t2 = System.currentTimeMillis();
-
- double time = (t2 - t1) / 1000.0;
-
- System.out.printf(
- "\t\ttablet: " + ke.getUUID() + " thread: " +
Thread.currentThread().getId()
- + " count: %,d cells time: %6.2f rate: %,6.2f cells/sec%n",
- count, time, count / time);
-
- finishCdl.countDown();
- }
-
- int getCount() {
- return count;
- }
-
- long getStartTime() {
- return t1;
- }
-
- long getFinishTime() {
- return t2;
- }
-
- }
-
- @SuppressFBWarnings(value = "DM_GC", justification = "gc is okay for test")
- private static void runTest(String desc, List<Test> tests, int numThreads,
- ExecutorService threadPool) throws Exception {
-
- System.out.println("\tRunning test : " + desc);
-
- CountDownLatch startSignal = new CountDownLatch(1);
- CountDownLatch finishedSignal = new CountDownLatch(numThreads);
-
- for (Test test : tests) {
- threadPool.execute(test);
- test.setSignals(startSignal, finishedSignal);
- }
-
- startSignal.countDown();
-
- finishedSignal.await();
-
- long minTime = Long.MAX_VALUE;
- long maxTime = Long.MIN_VALUE;
- long count = 0;
-
- for (Test test : tests) {
- minTime = Math.min(test.getStartTime(), minTime);
- maxTime = Math.max(test.getFinishTime(), maxTime);
- count += test.getCount();
- }
-
- double time = (maxTime - minTime) / 1000.0;
- System.out.printf("\tAggregate stats count: %,d cells time: %6.2f rate:
%,6.2f cells/sec%n",
- count, time, count / time);
- System.out.println();
-
- // run the gc between test so that object created during previous test are
not
- // collected in following test
- System.gc();
- System.gc();
- System.gc();
-
- }
-
- private static List<KeyExtent> findTablets(ClientContext context, boolean
selectLocalTablets,
- String tableName, SortedMap<KeyExtent,String> tabletLocations) throws
Exception {
-
- TableId tableId = context.getTableId(tableName);
- MetadataServicer.forTableId(context,
tableId).getTabletLocations(tabletLocations);
-
- InetAddress localaddress = InetAddress.getLocalHost();
-
- List<KeyExtent> candidates = new ArrayList<>();
-
- for (Entry<KeyExtent,String> entry : tabletLocations.entrySet()) {
- String loc = entry.getValue();
- if (loc != null) {
- boolean isLocal =
-
HostAndPort.fromString(entry.getValue()).getHost().equals(localaddress.getHostName());
-
- if (selectLocalTablets && isLocal) {
- candidates.add(entry.getKey());
- } else if (!selectLocalTablets && !isLocal) {
- candidates.add(entry.getKey());
- }
- }
- }
- return candidates;
- }
-
- private static List<KeyExtent> selectRandomTablets(int numThreads,
List<KeyExtent> candidates) {
- List<KeyExtent> tabletsToTest = new ArrayList<>();
-
- for (int i = 0; i < numThreads; i++) {
- int rindex = random.nextInt(candidates.size());
- tabletsToTest.add(candidates.get(rindex));
- Collections.swap(candidates, rindex, candidates.size() - 1);
- candidates = candidates.subList(0, candidates.size() - 1);
- }
- return tabletsToTest;
- }
-
- private static List<TabletFile> getTabletFiles(ServerContext context,
KeyExtent ke)
- throws IOException {
- return new ArrayList<>(
- MetadataTableUtil.getFileAndLogEntries(context,
ke).getSecond().keySet());
- }
-
- private static void reportHdfsBlockLocations(ServerContext context,
List<TabletFile> files)
- throws Exception {
- VolumeManager fs = context.getVolumeManager();
-
- System.out.println("\t\tFile block report : ");
- for (TabletFile file : files) {
- FileStatus status = fs.getFileStatus(file.getPath());
-
- if (status.isDirectory()) {
- // assume it is a map file
- status = fs.getFileStatus(new Path(file + "/data"));
- }
- FileSystem ns = fs.getFileSystemByPath(file.getPath());
- BlockLocation[] locs = ns.getFileBlockLocations(status, 0,
status.getLen());
-
- System.out.println("\t\t\tBlocks for : " + file);
-
- for (BlockLocation blockLocation : locs) {
- System.out.printf("\t\t\t\t offset : %,13d hosts :",
blockLocation.getOffset());
- for (String host : blockLocation.getHosts()) {
- System.out.print(" " + host);
- }
- System.out.println();
- }
- }
-
- System.out.println();
-
- }
-
- private static SortedKeyValueIterator<Key,Value>
createScanIterator(KeyExtent ke,
- Collection<SortedKeyValueIterator<Key,Value>> mapfiles, Authorizations
authorizations,
- byte[] defaultLabels, HashSet<Column> columnSet, List<IterInfo> ssiList,
- Map<String,Map<String,String>> ssio, boolean useTableIterators,
TableConfiguration conf,
- ServerContext context) throws IOException, ReflectiveOperationException {
-
- SortedMapIterator smi = new SortedMapIterator(new TreeMap<>());
-
- List<SortedKeyValueIterator<Key,Value>> iters = new
ArrayList<>(mapfiles.size() + 1);
-
- iters.addAll(mapfiles);
- iters.add(smi);
-
- MultiIterator multiIter = new MultiIterator(iters, ke.toDataRange());
- SortedKeyValueIterator<Key,Value> delIter =
- DeletingIterator.wrap(multiIter, false, Behavior.PROCESS);
- ColumnFamilySkippingIterator cfsi = new
ColumnFamilySkippingIterator(delIter);
- SortedKeyValueIterator<Key,Value> colFilter =
ColumnQualifierFilter.wrap(cfsi, columnSet);
- SortedKeyValueIterator<Key,Value> visFilter =
- VisibilityFilter.wrap(colFilter, authorizations, defaultLabels);
-
- if (useTableIterators) {
- var ibEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, ssiList,
ssio, conf);
- IteratorEnvironment iterEnv = new
SystemIteratorEnvironmentImpl.Builder(context)
- .withScope(IteratorScope.scan).withTableId(ke.tableId()).build();
- var iteratorBuilder = ibEnv.env(iterEnv).useClassLoader("test").build();
- return IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder);
- }
- return visFilter;
- }
-
- private static int readFiles(VolumeManager fs, AccumuloConfiguration aconf,
- List<TabletFile> files, KeyExtent ke, String[] columns) throws Exception
{
-
- int count = 0;
-
- HashSet<ByteSequence> columnSet = createColumnBSS(columns);
-
- for (TabletFile file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.getPath());
- FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
NoCryptoServiceFactory.NONE)
- .withTableConfiguration(aconf).build();
- Range range = new Range(ke.prevEndRow(), false, ke.endRow(), true);
- reader.seek(range, columnSet, !columnSet.isEmpty());
- while (reader.hasTop() && !range.afterEndKey(reader.getTopKey())) {
- count++;
- reader.next();
- }
- reader.close();
- }
-
- return count;
- }
-
- private static HashSet<ByteSequence> createColumnBSS(String[] columns) {
- HashSet<ByteSequence> columnSet = new HashSet<>();
- for (String c : columns) {
- columnSet.add(new ArrayByteSequence(c));
- }
- return columnSet;
- }
-
- private static int readFilesUsingIterStack(VolumeManager fs, ServerContext
context,
- List<TabletFile> files, Authorizations auths, KeyExtent ke, String[]
columns,
- boolean useTableIterators) throws Exception {
-
- SortedKeyValueIterator<Key,Value> reader;
-
- List<SortedKeyValueIterator<Key,Value>> readers = new
ArrayList<>(files.size());
-
- for (TabletFile file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.getPath());
- readers.add(FileOperations.getInstance().newReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
NoCryptoServiceFactory.NONE)
- .withTableConfiguration(context.getConfiguration()).build());
- }
-
- List<IterInfo> emptyIterinfo = Collections.emptyList();
- Map<String,Map<String,String>> emptySsio = Collections.emptyMap();
- TableConfiguration tconf = context.getTableConfiguration(ke.tableId());
- reader = createScanIterator(ke, readers, auths, new byte[] {}, new
HashSet<>(), emptyIterinfo,
- emptySsio, useTableIterators, tconf, context);
-
- HashSet<ByteSequence> columnSet = createColumnBSS(columns);
-
- reader.seek(new Range(ke.prevEndRow(), false, ke.endRow(), true),
columnSet,
- !columnSet.isEmpty());
-
- int count = 0;
-
- while (reader.hasTop()) {
- count++;
- reader.next();
- }
-
- return count;
-
- }
-
- private static int scanTablet(AccumuloClient client, String table,
Authorizations auths,
- Text prevEndRow, Text endRow, String[] columns) throws Exception {
-
- try (Scanner scanner = client.createScanner(table, auths)) {
- scanner.setRange(new Range(prevEndRow, false, endRow, true));
-
- for (String c : columns) {
- scanner.fetchColumnFamily(new Text(c));
- }
-
- int count = 0;
-
- for (Entry<Key,Value> entry : scanner) {
- if (entry != null) {
- count++;
- }
- }
- return count;
- }
- }
-
- private static void calcTabletStats(AccumuloClient client, String table,
Authorizations auths,
- KeyExtent ke, String[] columns) throws Exception {
-
- // long t1 = System.currentTimeMillis();
-
- try (Scanner scanner = client.createScanner(table, auths)) {
- scanner.setRange(new Range(ke.prevEndRow(), false, ke.endRow(), true));
-
- for (String c : columns) {
- scanner.fetchColumnFamily(new Text(c));
- }
-
- Stat rowLen = new Stat();
- Stat cfLen = new Stat();
- Stat cqLen = new Stat();
- Stat cvLen = new Stat();
- Stat valLen = new Stat();
- Stat colsPerRow = new Stat();
-
- Text lastRow = null;
- int colsPerRowCount = 0;
-
- for (Entry<Key,Value> entry : scanner) {
-
- Key key = entry.getKey();
- Text row = key.getRow();
-
- if (lastRow == null) {
- lastRow = row;
- }
-
- if (!lastRow.equals(row)) {
- colsPerRow.addStat(colsPerRowCount);
- lastRow = row;
- colsPerRowCount = 0;
- }
-
- colsPerRowCount++;
-
- rowLen.addStat(row.getLength());
- cfLen.addStat(key.getColumnFamilyData().length());
- cqLen.addStat(key.getColumnQualifierData().length());
- cvLen.addStat(key.getColumnVisibilityData().length());
- valLen.addStat(entry.getValue().get().length);
- }
-
- synchronized (System.out) {
- System.out.println("");
- System.out.println("\tTablet " + ke.getUUID() + " statistics : ");
- printStat("Row length", rowLen);
- printStat("Column family length", cfLen);
- printStat("Column qualifier length", cqLen);
- printStat("Column visibility length", cvLen);
- printStat("Value length", valLen);
- printStat("Columns per row", colsPerRow);
- System.out.println("");
- }
- }
- }
-
- private static void printStat(String desc, Stat s) {
- System.out.printf("\t\tDescription: [%30s] average: %,6.2f min: %,d max:
%,d %n", desc,
- s.mean(), s.min(), s.max());
-
- }
-
-}
diff --git a/test/src/main/resources/log4j2-test.properties
b/test/src/main/resources/log4j2-test.properties
index 9ffb22da16..bd810eaec0 100644
--- a/test/src/main/resources/log4j2-test.properties
+++ b/test/src/main/resources/log4j2-test.properties
@@ -18,13 +18,13 @@
#
dest = err
-name = AccumuloITLoggingProperties
+name = AccumuloTestLoggingProperties
appender.console.type = Console
appender.console.name = STDOUT
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} %T [%c{2}] %-5p: %m%n
+appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p:
%m%n
logger.01.name = org.apache.accumulo.core
logger.01.level = debug
@@ -160,4 +160,3 @@ logger.41.level = info
rootLogger.level = debug
rootLogger.appenderRef.console.ref = STDOUT
-
diff --git
a/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java
b/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java
deleted file mode 100644
index 03cebd2cb8..0000000000
---
a/test/src/test/java/org/apache/accumulo/test/performance/scan/CollectTabletStatsTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.test.performance.scan;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import org.junit.jupiter.api.Test;
-
-public class CollectTabletStatsTest {
-
- @Test
- public void paramsDefaulThreadTest() {
- String tablename = "aTable";
- String[] args = {"-t", tablename, "--iterations", "2"};
-
- final CollectTabletStats.CollectOptions opts = new
CollectTabletStats.CollectOptions();
- opts.parseArgs(CollectTabletStats.class.getName(), args);
-
- assertEquals(2, opts.iterations, "Check iterations is set, default is 3");
- assertEquals(0, tablename.compareTo(opts.tableName), "Check tablename is
set");
- assertEquals(1, opts.numThreads, "Check default numThreads");
- }
-
- @Test
- public void paramsSetThreadsTest() {
- String tablename = "aTable";
- String[] args = {"-t", tablename, "--iterations", "2", "--numThreads",
"99"};
-
- final CollectTabletStats.CollectOptions opts = new
CollectTabletStats.CollectOptions();
- opts.parseArgs(CollectTabletStats.class.getName(), args);
-
- assertEquals(2, opts.iterations, "Check iterations is set, default is 3");
- assertEquals(0, tablename.compareTo(opts.tableName), "Check tablename is
set");
- assertEquals(99, opts.numThreads, "Check numThreads is set");
- }
-}
