This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit af4955d592de931762e7c774643aca626bfab964 Merge: c27dbe7823 370f78bdd2 Author: Christopher Tubbs <[email protected]> AuthorDate: Mon Mar 9 23:09:38 2026 -0400 Merge branch '2.1' assemble/conf/log4j2-service.properties | 6 +- assemble/conf/log4j2.properties | 3 +- assemble/pom.xml | 1 + core/pom.xml | 1 + .../classloader/URLContextClassLoaderFactory.java | 8 +- .../core/clientImpl/ClientTabletCacheImpl.java | 23 +- .../core/clientImpl/NamespaceOperationsImpl.java | 11 +- .../core/clientImpl/RootClientTabletCache.java | 5 +- .../core/clientImpl/TableOperationsImpl.java | 10 +- .../TabletServerBatchReaderIterator.java | 20 +- .../core/clientImpl/TabletServerBatchWriter.java | 7 +- .../accumulo/core/clientImpl/ThriftScanner.java | 16 +- .../metadata/MetadataCachedTabletObtainer.java | 11 +- .../apache/accumulo/core/util/threads/Threads.java | 15 + core/src/test/resources/log4j2-test.properties | 4 +- hadoop-mapreduce/pom.xml | 1 + .../src/test/resources/log4j2-test.properties | 4 +- iterator-test-harness/pom.xml | 1 + .../src/test/resources/log4j2-test.properties | 5 +- minicluster/pom.xml | 1 + .../miniclusterImpl/MiniAccumuloClusterImpl.java | 2 - .../src/test/resources/log4j2-test.properties | 5 +- pom.xml | 1 + .../base/src/test/resources/log4j2-test.properties | 4 +- .../gc/src/test/resources/log4j2-test.properties | 5 +- .../src/test/resources/log4j2-test.properties | 5 +- .../java/org/apache/accumulo/monitor/Monitor.java | 10 +- .../src/test/resources/log4j2-test.properties | 5 +- .../src/test/resources/log4j2-test.properties | 4 +- shell/pom.xml | 1 + shell/src/test/resources/log4j2-test.properties | 4 +- start/pom.xml | 1 + .../apache/accumulo/start/util/MiniDFSUtil.java | 53 -- start/src/test/resources/log4j2-test.properties | 4 +- test/pom.xml | 1 + .../accumulo/test/CountNameNodeOpsBulkIT.java | 4 +- .../ThriftServerBindsBeforeZooKeeperLockIT.java | 4 +- .../accumulo/test/functional/MonitorSslIT.java | 4 +- .../accumulo/test/functional/ReadWriteIT.java | 4 +- .../test/performance/scan/CollectTabletStats.java | 641 --------------------- .../apache/accumulo/test/start/KeywordStartIT.java | 3 - test/src/main/resources/log4j2-test.properties | 4 +- .../performance/scan/CollectTabletStatsTest.java | 52 -- 43 files changed, 113 insertions(+), 861 deletions(-) diff --cc assemble/conf/log4j2-service.properties index e2e49de5d4,f96e472252..a080a7d4ad --- a/assemble/conf/log4j2-service.properties +++ b/assemble/conf/log4j2-service.properties @@@ -32,9 -32,9 +32,9 @@@ appender.console.type = Consol appender.console.name = STDERR appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout - appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n + appender.console.layout.pattern = %d{ISO8601} Thread[%T] [%-8c{2}] %-5p: %m%n appender.console.filter.threshold.type = ThresholdFilter -appender.console.filter.threshold.level = info +appender.console.filter.threshold.level = error appender.rolling.type = RollingFile appender.rolling.name = LogFiles @@@ -51,21 -51,28 +51,21 @@@ appender.rolling.policies.size.size=512 appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.max = 10 -appender.audit.type = RollingFile -appender.audit.name = AuditLogFiles -appender.audit.fileName = ${filename}.audit -appender.audit.filePattern = ${filename}-%d{yyyy-MM-dd}-%i.audit.gz -appender.audit.layout.type = PatternLayout -appender.audit.layout.pattern = %d{ISO8601} Thread[%T] [%-8c{2}] %-5p: %m%n -appender.audit.policies.type = Policies -appender.audit.policies.time.type = TimeBasedTriggeringPolicy -appender.audit.policies.time.interval = 1 -appender.audit.policies.time.modulate = true -appender.audit.policies.size.type = SizeBasedTriggeringPolicy -appender.audit.policies.size.size=512MB -appender.audit.strategy.type = DefaultRolloverStrategy -appender.audit.strategy.max = 10 - -appender.monitor.type = AccumuloMonitor -appender.monitor.name = MonitorLog -appender.monitor.filter.threshold.type = ThresholdFilter -appender.monitor.filter.threshold.level = warn -#appender.monitor.async = true -#appender.monitor.maxThreads = 2 -#appender.monitor.queueSize = 1024 +# uncomment for separate audit logs +#appender.audit.type = RollingFile +#appender.audit.name = AuditLogFiles +#appender.audit.fileName = ${filename}.audit +#appender.audit.filePattern = ${filename}-%d{yyyy-MM-dd}-%i.audit.gz +#appender.audit.layout.type = PatternLayout - #appender.audit.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n ++#appender.audit.layout.pattern = %d{ISO8601} Thread[%T] [%-8c{2}] %-5p: %m%n +#appender.audit.policies.type = Policies +#appender.audit.policies.time.type = TimeBasedTriggeringPolicy +#appender.audit.policies.time.interval = 1 +#appender.audit.policies.time.modulate = true +#appender.audit.policies.size.type = SizeBasedTriggeringPolicy +#appender.audit.policies.size.size=512MB +#appender.audit.strategy.type = DefaultRolloverStrategy +#appender.audit.strategy.max = 10 logger.zookeeper.name = org.apache.zookeeper logger.zookeeper.level = error diff --cc assemble/pom.xml index 295128c008,ecd9db9444..4db8052fd1 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo</artifactId> <packaging>pom</packaging> diff --cc core/pom.xml index b35298d379,02ecf146ba..fa9ac86194 --- a/core/pom.xml +++ b/core/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-core</artifactId> <name>Apache Accumulo Core</name> diff --cc core/src/main/java/org/apache/accumulo/core/classloader/URLContextClassLoaderFactory.java index 9e88945756,0000000000..32681e6e5d mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/classloader/URLContextClassLoaderFactory.java +++ b/core/src/main/java/org/apache/accumulo/core/classloader/URLContextClassLoaderFactory.java @@@ -1,67 -1,0 +1,69 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.classloader; + ++import java.io.UncheckedIOException; +import java.net.MalformedURLException; ++import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; + +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory; +import org.apache.accumulo.core.util.cache.Caches; +import org.apache.accumulo.core.util.cache.Caches.CacheName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.benmanes.caffeine.cache.Cache; + +/** + * The default implementation of ContextClassLoaderFactory. This classloader returns a + * URLClassLoader based on the given context value which is a CSV list of URLs. For example, + * file://path/one/jar1.jar,file://path/two/jar2.jar + */ +public class URLContextClassLoaderFactory implements ContextClassLoaderFactory { + + private static final Logger LOG = LoggerFactory.getLogger(URLContextClassLoaderFactory.class); + + // Cache the class loaders for re-use + // WeakReferences are used so that the class loaders can be cleaned up when no longer needed + // Classes that are loaded contain a reference to the class loader used to load them + // so the class loader will be garbage collected when no more classes are loaded that reference it + private final Cache<String,URLClassLoader> classloaders = + Caches.getInstance().createNewBuilder(CacheName.CLASSLOADERS, true).weakValues().build(); + + @Override + public ClassLoader getClassLoader(String context) { + if (context == null) { + throw new IllegalArgumentException("Unknown context"); + } + + return classloaders.get(context, k -> { + LOG.debug("Creating URLClassLoader for context, uris: {}", context); - return new URLClassLoader(Arrays.stream(context.split(",")).map(url -> { ++ return new URLClassLoader(Arrays.stream(context.split(",")).map(p -> { + try { - return new URL(url); ++ return URI.create(p).toURL(); + } catch (MalformedURLException e) { - throw new RuntimeException(e); ++ throw new UncheckedIOException(e); + } + }).toArray(URL[]::new), ClassLoader.getSystemClassLoader()); + }); + } +} diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java index 391901246d,0000000000..2ffbed12eb mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java @@@ -1,873 -1,0 +1,870 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.clientImpl; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.SortedMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.InvalidTabletHostingRequestException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent; +import org.apache.accumulo.core.manager.state.tables.TableState; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.LockMap; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.core.util.TextUtil; +import org.apache.accumulo.core.util.Timer; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + +/** + * This class has two concurrency goals. First when a thread request data that is currently present + * in the cache, it should never block. Second when a thread request data that is not in the cache + * only one lookup per metadata tablet will happen concurrently. The purpose of this second goal is + * to avoid redundant concurrent metadata lookups in a client process. + * + * <p> + * The first goal is achieved by using a ConcurrentSkipListMap to store the caches data making it + * safe for multiple threads to read and write to the map. The second goal is achieved by using a + * {@link LockMap} keyed on metadata table extents when doing metadata table lookups. + * + * <p> + * Below is an example of how this cache is intended to work. + * + * <ol> + * <li>Thread_1 lookups up row A that is not currently present in the cache. + * <li>Thread_2 lookups up row C that is not currently present in the cache. + * <li>Thread_3 lookups up row Q that is not currently present in the cache. + * <li>Thread_1 finds metadata tablet MT1 stores information on row A and locks the extent for MT1. + * <li>Thread_2 finds metadata tablet MT1 stores information on row C and locks the extent for MT1. + * <li>Thread_3 finds metadata tablet MT2 stores information on row Q and locks the extent for MT2. + * <li>Thread_1 acquires the lock for MT1 + * <li>Thread_2 blocks waiting to lock MT1 + * <li>Thread_3 acquires the lock for MT2 + * <li>Thread_4 finds row Z in the cache and immediately returns its user tablet information. If + * this data was not cached, it would have needed to read metadata tablet MT2 which is currently + * locked and would have blocked. + * <li>Thread_1 reads user_tablet_1_metadata that contains row A from MT1 and adds it to the cache. + * It also opportunistically reads a few more user tablets metadata from MT1 after the first user + * tablet adds them the cache. + * <li>Thread_3 reads user_tablet_10_metadata that contains row Q from MT2 and adds it to the cache. + * <li>Thread_1 finds user_tablet_1_metadata in the cache and returns it as the tablet for row A. + * <li>Thread_1 unlocks the lock for MT1 + * <li>Thread_3 finds user_tablet_10_metadata in the cache and returns it as the tablet for row Q. + * <li>Thread_3 unlocks the lock for MT2 + * <li>Thread_2 acquires the lock for MT1 + * <li>Thread_2 checks the cache and finds the information it needs is now present in the cache + * because it was found by Thread_1. No metadata lookup is done, the information from the cache is + * returned. + * <li>Thread_2 unlocks the lock for MT1 + * </ol> + * + */ +public class ClientTabletCacheImpl extends ClientTabletCache { + + private static final Logger log = LoggerFactory.getLogger(ClientTabletCacheImpl.class); + private static final AtomicBoolean HOSTING_ENABLED = new AtomicBoolean(true); + + // MAX_TEXT represents a TEXT object that is greater than all others. Attempted to use null for + // this purpose, but there seems to be a bug in TreeMap.tailMap with null. Therefore instead of + // using null, created MAX_TEXT. + static final Text MAX_TEXT = new Text(); + + static final Comparator<Text> END_ROW_COMPARATOR = (o1, o2) -> { + if (o1 == o2) { + return 0; + } + if (o1 == MAX_TEXT) { + return 1; + } + if (o2 == MAX_TEXT) { + return -1; + } + return o1.compareTo(o2); + }; + + protected final TableId tableId; + protected final ClientTabletCache parent; + protected final ConcurrentSkipListMap<Text,CachedTablet> metaCache = + new ConcurrentSkipListMap<>(END_ROW_COMPARATOR); + protected final CachedTabletObtainer tabletObtainer; + private final TabletServerLockChecker lockChecker; + protected final Text lastTabletRow; + + private final AtomicLong tabletHostingRequestCount = new AtomicLong(0); + + private final LockMap<KeyExtent> lookupLocks = new LockMap<>(); + + public interface CachedTabletObtainer { + /** + * @return null when unable to read information successfully + */ + CachedTablets lookupTablet(ClientContext context, CachedTablet src, Text row, Text stopRow) + throws AccumuloSecurityException, AccumuloException; + } + + public interface TabletServerLockChecker { + boolean isLockHeld(String tserver, String session); + + void invalidateCache(String server); + } + + private class LockCheckerSession { + + private final HashSet<Pair<String,String>> okLocks = new HashSet<>(); + private final HashSet<Pair<String,String>> invalidLocks = new HashSet<>(); + + private CachedTablet checkLock(CachedTablet tl) { + // the goal of this class is to minimize calls out to lockChecker under that + // assumption that + // it is a resource synchronized among many threads... want to + // avoid fine-grained synchronization when binning lots of mutations or ranges... remember + // decisions from the lockChecker in thread local unsynchronized + // memory + + if (tl == null) { + return null; + } + + if (tl.getTserverLocation().isEmpty()) { + return tl; + } + + Pair<String,String> lock = + new Pair<>(tl.getTserverLocation().orElseThrow(), tl.getTserverSession().orElseThrow()); + + if (okLocks.contains(lock)) { + return tl; + } + + if (invalidLocks.contains(lock)) { + return null; + } + + if (lockChecker.isLockHeld(tl.getTserverLocation().orElseThrow(), + tl.getTserverSession().orElseThrow())) { + okLocks.add(lock); + return tl; + } + + if (log.isTraceEnabled()) { + log.trace("Tablet server {} {} no longer holds its lock", tl.getTserverLocation(), + tl.getTserverSession()); + } + + invalidLocks.add(lock); + + return null; + } + } + + public ClientTabletCacheImpl(TableId tableId, ClientTabletCache parent, CachedTabletObtainer tlo, + TabletServerLockChecker tslc) { + this.tableId = tableId; + this.parent = parent; + this.tabletObtainer = tlo; + this.lockChecker = tslc; + + this.lastTabletRow = new Text(tableId.canonical()); + lastTabletRow.append(new byte[] {'<'}, 0, 1); + } + + @Override + public <T extends Mutation> void binMutations(ClientContext context, List<T> mutations, + Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures) + throws AccumuloException, AccumuloSecurityException, TableNotFoundException, + InvalidTabletHostingRequestException { + + Timer timer = null; + + if (log.isTraceEnabled()) { - log.trace("tid={} Binning {} mutations for table {}", Thread.currentThread().getId(), - mutations.size(), tableId); ++ log.trace("Binning {} mutations for table {}", mutations.size(), tableId); + timer = Timer.startNew(); + } + + ArrayList<T> notInCache = new ArrayList<>(); + Text row = new Text(); + + LockCheckerSession lcSession = new LockCheckerSession(); + + // for this to be efficient rows need to be in sorted order, but always sorting is slow... + // therefore only sort the + // stuff not in the cache.... it is most efficient to pass _locateTablet rows in sorted order + + // For this to be efficient, need to avoid fine grained synchronization and fine grained + // logging. + // Therefore methods called by this are not synchronized and should not log. + + for (T mutation : mutations) { + row.set(mutation.getRow()); + CachedTablet tl = findTabletInCache(row); + if (!addMutation(binnedMutations, mutation, tl, lcSession)) { + notInCache.add(mutation); + } + } + + HashSet<CachedTablet> locationLess = new HashSet<>(); + + if (!notInCache.isEmpty()) { + notInCache.sort((o1, o2) -> WritableComparator.compareBytes(o1.getRow(), 0, + o1.getRow().length, o2.getRow(), 0, o2.getRow().length)); + + // Want to ignore any entries in the cache w/o a location that were created before the + // following time. Entries created after the following time may have been populated by the + // following loop, and we want to use those. + Timer cacheCutoffTimer = Timer.startNew(); + + for (T mutation : notInCache) { + + row.set(mutation.getRow()); + + CachedTablet tl = + _findTablet(context, row, false, lcSession, LocationNeed.REQUIRED, cacheCutoffTimer); + + if (!addMutation(binnedMutations, mutation, tl, lcSession)) { + failures.add(mutation); + if (tl != null && tl.getTserverLocation().isEmpty()) { + locationLess.add(tl); + } + } + } + } + + requestTabletHosting(context, locationLess); + + if (timer != null) { - log.trace("tid={} Binned {} mutations for table {} to {} tservers in {}", - Thread.currentThread().getId(), mutations.size(), tableId, binnedMutations.size(), - String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); ++ log.trace("Binned {} mutations for table {} to {} tservers in {}", mutations.size(), tableId, ++ binnedMutations.size(), String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); + } + + } + + private <T extends Mutation> boolean addMutation( + Map<String,TabletServerMutations<T>> binnedMutations, T mutation, CachedTablet tl, + LockCheckerSession lcSession) { + + if (tl == null || tl.getTserverLocation().isEmpty()) { + return false; + } + + TabletServerMutations<T> tsm = binnedMutations.get(tl.getTserverLocation().orElseThrow()); + + if (tsm == null) { + // do lock check once per tserver here to make binning faster + boolean lockHeld = lcSession.checkLock(tl) != null; + if (lockHeld) { + tsm = new TabletServerMutations<>(tl.getTserverSession().orElseThrow()); + binnedMutations.put(tl.getTserverLocation().orElseThrow(), tsm); + } else { + return false; + } + } + + // its possible the same tserver could be listed with different sessions + if (tsm.getSession().equals(tl.getTserverSession().orElseThrow())) { + tsm.addMutation(tl.getExtent(), mutation); + return true; + } + + return false; + } + + static boolean isContiguous(List<CachedTablet> cachedTablets) { + + Iterator<CachedTablet> iter = cachedTablets.iterator(); + KeyExtent prevExtent = iter.next().getExtent(); + + while (iter.hasNext()) { + KeyExtent currExtent = iter.next().getExtent(); + + if (!currExtent.isPreviousExtent(prevExtent)) { + return false; + } + + prevExtent = currExtent; + } + + return true; + } + + private List<Range> findTablets(ClientContext context, List<Range> ranges, + BiConsumer<CachedTablet,Range> rangeConsumer, boolean useCache, LockCheckerSession lcSession, + LocationNeed locationNeed, Consumer<CachedTablet> locationlessConsumer) + throws AccumuloException, AccumuloSecurityException, TableNotFoundException, + InvalidTabletHostingRequestException { + List<Range> failures = new ArrayList<>(); + List<CachedTablet> cachedTablets = new ArrayList<>(); + + // Use anything in the cache w/o a location populated after this point in time. Cache entries + // w/o a location created before the following time should be ignored and the metadata table + // consulted. + Timer cacheCutoffTimer = Timer.startNew(); + + l1: for (Range range : ranges) { + + cachedTablets.clear(); + + Text startRow; + + if (range.getStartKey() != null) { + startRow = range.getStartKey().getRow(); + } else { + startRow = new Text(); + } + + CachedTablet tl = null; + + if (useCache) { + tl = lcSession.checkLock(findTabletInCache(startRow)); + } else { + tl = _findTablet(context, startRow, false, lcSession, locationNeed, cacheCutoffTimer); + } + + if (tl == null) { + failures.add(range); + continue; + } + + cachedTablets.add(tl); + + // a range may extend over multiple tablets, look for additional tablet that overlap the range + while (tl.getExtent().endRow() != null + && !range.afterEndKey(new Key(tl.getExtent().endRow()).followingKey(PartialKey.ROW))) { + if (useCache) { + Text row = new Text(tl.getExtent().endRow()); + row.append(new byte[] {0}, 0, 1); + tl = lcSession.checkLock(findTabletInCache(row)); + } else { + tl = _findTablet(context, tl.getExtent().endRow(), true, lcSession, locationNeed, + cacheCutoffTimer); + } + + if (tl == null) { + failures.add(range); + continue l1; + } + cachedTablets.add(tl); + } + + // pass all tablets without a location before failing range + cachedTablets.stream().filter(tloc -> tloc.getTserverLocation().isEmpty()) + .forEach(locationlessConsumer); + + if (locationNeed == LocationNeed.REQUIRED + && !cachedTablets.stream().allMatch(tloc -> tloc.getTserverLocation().isPresent())) { + failures.add(range); + continue; + } + + // Ensure the extents found are non overlapping and have no holes. When reading some extents + // from the cache and other from the metadata table in the loop above we may end up with + // non-contiguous extents. This can happen when a subset of exents are placed in the cache and + // then after that merges and splits happen. + if (isContiguous(cachedTablets)) { + for (CachedTablet tl2 : cachedTablets) { + rangeConsumer.accept(tl2, range); + } + } else { + failures.add(range); + } + + } + + return failures; + } + + @Override + public List<Range> findTablets(ClientContext context, List<Range> ranges, + BiConsumer<CachedTablet,Range> rangeConsumer, LocationNeed locationNeed) + throws AccumuloException, AccumuloSecurityException, TableNotFoundException, + InvalidTabletHostingRequestException { + + /* + * For this to be efficient, need to avoid fine grained synchronization and fine grained + * logging. Therefore methods called by this are not synchronized and should not log. + */ + + Timer timer = null; + + if (log.isTraceEnabled()) { - log.trace("tid={} Binning {} ranges for table {}", Thread.currentThread().getId(), - ranges.size(), tableId); ++ log.trace("Binning {} ranges for table {}", ranges.size(), tableId); + timer = Timer.startNew(); + } + + LockCheckerSession lcSession = new LockCheckerSession(); + + List<Range> failures; + // for this to be optimal, need to look ranges up in sorted order when + // ranges are not present in cache... however do not want to always + // sort ranges... therefore try binning ranges using only the cache + // and sort whatever fails and retry + failures = + findTablets(context, ranges, rangeConsumer, true, lcSession, locationNeed, keyExtent -> {}); + + if (!failures.isEmpty()) { + // sort failures by range start key + Collections.sort(failures); + + // use a hashset because some ranges may overlap the same extent, so want to avoid duplicate + // extents + HashSet<CachedTablet> locationLess = new HashSet<>(); + Consumer<CachedTablet> locationLessConsumer; + if (locationNeed == LocationNeed.REQUIRED) { + locationLessConsumer = locationLess::add; + } else { + locationLessConsumer = keyExtent -> {}; + } + + // try lookups again + failures = findTablets(context, failures, rangeConsumer, false, lcSession, locationNeed, + locationLessConsumer); + + requestTabletHosting(context, locationLess); + } + + if (timer != null) { - log.trace("tid={} Binned {} ranges for table {} in {}", Thread.currentThread().getId(), - ranges.size(), tableId, String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); ++ log.trace("Binned {} ranges for table {} in {}", ranges.size(), tableId, ++ String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); + } + + return failures; + } + + @Override + public void invalidateCache(KeyExtent failedExtent) { + removeOverlapping(metaCache, failedExtent); + if (log.isTraceEnabled()) { + log.trace("Invalidated extent={}", failedExtent); + } + } + + @Override + public void invalidateCache(Collection<KeyExtent> keySet) { + keySet.forEach(extent -> removeOverlapping(metaCache, extent)); + if (log.isTraceEnabled()) { + log.trace("Invalidated {} cache entries for table {}", keySet.size(), tableId); + } + } + + @Override + public void invalidateCache() { + int invalidatedCount = metaCache.size(); + metaCache.clear(); + this.tabletHostingRequestCount.set(0); + if (log.isTraceEnabled()) { + log.trace("invalidated all {} cache entries for table={}", invalidatedCount, tableId); + } + } + + @Override + public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow, + LocationNeed locationNeed, int minimumHostAhead, Range hostAheadRange) + throws AccumuloException, AccumuloSecurityException, TableNotFoundException, + InvalidTabletHostingRequestException { + + Timer timer = null; + + if (log.isTraceEnabled()) { - log.trace("tid={} Locating tablet table={} row={} skipRow={}", - Thread.currentThread().getId(), tableId, TextUtil.truncate(row), skipRow); ++ log.trace("Locating tablet table={} row={} skipRow={}", tableId, TextUtil.truncate(row), ++ skipRow); + timer = Timer.startNew(); + } + + LockCheckerSession lcSession = new LockCheckerSession(); + CachedTablet tl = _findTablet(context, row, skipRow, lcSession, locationNeed, Timer.startNew()); + + if (timer != null) { - log.trace("tid={} Located tablet {} at {} in {}", Thread.currentThread().getId(), - (tl == null ? "null" : tl.getExtent()), (tl == null ? "null" : tl.getTserverLocation()), ++ log.trace("Located tablet {} at {} in {}", (tl == null ? "null" : tl.getExtent()), ++ (tl == null ? "null" : tl.getTserverLocation()), + String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); + } + + if (tl != null && locationNeed == LocationNeed.REQUIRED) { + // Look at the next (minimumHostAhead * 2) tablets and return which ones need hosting. See the + // javadoc in the superclass of this method for more details. + Map<KeyExtent,CachedTablet> extentsToHost = findExtentsToHost(context, minimumHostAhead * 2, + hostAheadRange, lcSession, tl, locationNeed); + + if (!extentsToHost.isEmpty()) { + if (extentsToHost.containsKey(tl.getExtent()) || extentsToHost.size() >= minimumHostAhead) { + requestTabletHosting(context, extentsToHost.values()); + } + } + + if (tl.getTserverLocation().isEmpty()) { + return null; + } + } + + return tl; + + } + + private Map<KeyExtent,CachedTablet> findExtentsToHost(ClientContext context, int hostAheadCount, + Range hostAheadRange, LockCheckerSession lcSession, CachedTablet firstTablet, + LocationNeed locationNeed) throws AccumuloException, TableNotFoundException, + InvalidTabletHostingRequestException, AccumuloSecurityException { + + // its only expected that this method is called when location need is required + Preconditions.checkArgument(locationNeed == LocationNeed.REQUIRED); + + Map<KeyExtent,CachedTablet> extentsToHost; + + if (hostAheadCount > 0) { + extentsToHost = new HashMap<>(); + if (firstTablet.getTserverLocation().isEmpty()) { + extentsToHost.put(firstTablet.getExtent(), firstTablet); + } + + KeyExtent extent = firstTablet.getExtent(); + + var currTablet = extent; + + // Use anything in the cache w/o a location populated after this point in time. Cache entries + // w/o a location created before the following time should be ignored and the metadata table + // consulted. + Timer cacheCutoffTimer = Timer.startNew(); + + for (int i = 0; i < hostAheadCount; i++) { + if (currTablet.endRow() == null || hostAheadRange + .afterEndKey(new Key(currTablet.endRow()).followingKey(PartialKey.ROW))) { + break; + } + + CachedTablet followingTablet = _findTablet(context, currTablet.endRow(), true, lcSession, + locationNeed, cacheCutoffTimer); + + if (followingTablet == null) { + break; + } + + currTablet = followingTablet.getExtent(); + + if (followingTablet.getTserverLocation().isEmpty() + && !followingTablet.wasHostingRequested()) { + extentsToHost.put(followingTablet.getExtent(), followingTablet); + } + } + } else if (firstTablet.getTserverLocation().isEmpty()) { + extentsToHost = Map.of(firstTablet.getExtent(), firstTablet); + } else { + extentsToHost = Map.of(); + } + return extentsToHost; + } + + @Override + public long getTabletHostingRequestCount() { + return tabletHostingRequestCount.get(); + } + + @VisibleForTesting + public void enableTabletHostingRequests(boolean enabled) { + HOSTING_ENABLED.set(enabled); + } + + private static final Duration STALE_DURATION = Duration.ofMinutes(2); + + private void requestTabletHosting(ClientContext context, + Collection<CachedTablet> tabletsWithNoLocation) throws AccumuloException, + AccumuloSecurityException, TableNotFoundException, InvalidTabletHostingRequestException { + + if (!HOSTING_ENABLED.get()) { + return; + } + + // System tables should always be hosted + if (SystemTables.containsTableId(tableId)) { + return; + } + + if (tabletsWithNoLocation.isEmpty()) { + return; + } + + if (context.getTableState(tableId) != TableState.ONLINE) { + log.trace("requestTabletHosting: table {} is not online", tableId); + return; + } + + List<TKeyExtent> extentsToBringOnline = new ArrayList<>(); + for (var cachedTablet : tabletsWithNoLocation) { + if (cachedTablet.getCreationTimer().elapsed().compareTo(STALE_DURATION) < 0) { + if (cachedTablet.getAvailability() == TabletAvailability.ONDEMAND) { + if (!cachedTablet.wasHostingRequested()) { + extentsToBringOnline.add(cachedTablet.getExtent().toThrift()); + log.trace("requesting ondemand tablet to be hosted {}", cachedTablet.getExtent()); + } else { + log.trace("ignoring ondemand tablet that already has a hosting request in place {} {}", + cachedTablet.getExtent(), cachedTablet.getCreationTimer().elapsed()); + } + } else if (cachedTablet.getAvailability() == TabletAvailability.UNHOSTED) { + throw new InvalidTabletHostingRequestException("Extent " + cachedTablet.getExtent() + + " has a tablet availability " + TabletAvailability.UNHOSTED); + } + } else { + // When a tablet does not have a location it is reread from the metadata table before this + // method is called. Therefore, it's expected that entries in the cache are recent. If the + // entries are not recent it could have two causes. One is a bug in the Accumulo code. + // Another is externalities like process swapping or slow metadata table reads. Logging a + // warning in case there is a bug. If the warning ends up being too spammy and is caused by + // externalities then this code/warning will need to be improved. + log.warn("Unexpected stale tablet seen in cache {}", cachedTablet.getExtent()); + invalidateCache(cachedTablet.getExtent()); + } + } + + if (!extentsToBringOnline.isEmpty()) { + log.debug("Requesting hosting for {} ondemand tablets for table id {}.", + extentsToBringOnline.size(), tableId); + ThriftClientTypes.MANAGER.executeVoid(context, + client -> client.requestTabletHosting(TraceUtil.traceInfo(), context.rpcCreds(), + tableId.canonical(), extentsToBringOnline)); + tabletHostingRequestCount.addAndGet(extentsToBringOnline.size()); + } + } + + private void lookupTablet(ClientContext context, Text row, LockCheckerSession lcSession, + CachedTablet before) throws AccumuloException, AccumuloSecurityException, + TableNotFoundException, InvalidTabletHostingRequestException { + Text metadataRow = new Text(tableId.canonical()); + metadataRow.append(new byte[] {';'}, 0, 1); + metadataRow.append(row.getBytes(), 0, row.getLength()); + CachedTablet ptl = parent.findTablet(context, metadataRow, false, LocationNeed.REQUIRED); + + if (ptl == null) { + return; + } + + try (var unused = lookupLocks.lock(ptl.getExtent())) { + // Now that the lock is acquired, detect if another thread populated cache since the last time + // the cache was read. If so then do not need to read from metadata store. + CachedTablet after = findTabletInCache(row); + if (after != null && after != before && lcSession.checkLock(after) != null) { + return; + } + // Lookup tablets in metadata table and update cache. Also updating the cache while holding + // the lock is important as it ensures other threads that are waiting on the lock will see + // what this thread found and may be able to avoid metadata lookups. + lookupTablet(context, lcSession, ptl, metadataRow); + } + } + + private void lookupTablet(ClientContext context, LockCheckerSession lcSession, CachedTablet ptl, + Text metadataRow) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, + InvalidTabletHostingRequestException { + CachedTablets cachedTablets = + tabletObtainer.lookupTablet(context, ptl, metadataRow, lastTabletRow); + if (cachedTablets == null) { + parent.invalidateCache(ptl.getExtent()); + } + while (cachedTablets != null && cachedTablets.getCachedTablets().isEmpty()) { + // try the next tablet, the current tablet does not have any tablets that overlap the row + Text er = ptl.getExtent().endRow(); + if (er != null && er.compareTo(lastTabletRow) < 0) { + // System.out.println("er "+er+" ltr "+lastTabletRow); + ptl = parent.findTablet(context, er, true, LocationNeed.REQUIRED); + if (ptl != null) { + cachedTablets = tabletObtainer.lookupTablet(context, ptl, metadataRow, lastTabletRow); + if (cachedTablets == null) { + parent.invalidateCache(ptl.getExtent()); + } + } else { + break; + } + } else { + break; + } + } + + if (cachedTablets == null) { + return; + } + + // cannot assume the list contains contiguous key extents... so it is probably + // best to deal with each extent individually + + Text lastEndRow = null; + for (CachedTablet cachedTablet : cachedTablets.getCachedTablets()) { + + KeyExtent ke = cachedTablet.getExtent(); + CachedTablet locToCache; + + // create new location if current prevEndRow == endRow + if ((lastEndRow != null) && (ke.prevEndRow() != null) && ke.prevEndRow().equals(lastEndRow)) { + locToCache = new CachedTablet(new KeyExtent(ke.tableId(), ke.endRow(), lastEndRow), + cachedTablet.getTserverLocation(), cachedTablet.getTserverSession(), + cachedTablet.getAvailability(), cachedTablet.wasHostingRequested()); + } else { + locToCache = cachedTablet; + } + + // save endRow for next iteration + lastEndRow = locToCache.getExtent().endRow(); + + updateCache(locToCache, lcSession); + } + } + + private void updateCache(CachedTablet cachedTablet, LockCheckerSession lcSession) { + if (!cachedTablet.getExtent().tableId().equals(tableId)) { + // sanity check + throw new IllegalStateException( + "Unexpected extent returned " + tableId + " " + cachedTablet.getExtent()); + } + + // clear out any overlapping extents in cache + removeOverlapping(metaCache, cachedTablet.getExtent()); + + // do not add to cache unless lock is held + if (lcSession.checkLock(cachedTablet) == null) { + return; + } + + // add it to cache + Text er = cachedTablet.getExtent().endRow(); + if (er == null) { + er = MAX_TEXT; + } + metaCache.put(er, cachedTablet); + } + + static void removeOverlapping(NavigableMap<Text,CachedTablet> metaCache, KeyExtent nke) { + Iterator<Entry<Text,CachedTablet>> iter; + + if (nke.prevEndRow() == null) { + iter = metaCache.entrySet().iterator(); + } else { + Text row = rowAfterPrevRow(nke); + SortedMap<Text,CachedTablet> tailMap = metaCache.tailMap(row); + iter = tailMap.entrySet().iterator(); + } + + while (iter.hasNext()) { + Entry<Text,CachedTablet> entry = iter.next(); + + KeyExtent ke = entry.getValue().getExtent(); + + if (stopRemoving(nke, ke)) { + break; + } + + iter.remove(); + } + } + + private static boolean stopRemoving(KeyExtent nke, KeyExtent ke) { + return ke.prevEndRow() != null && nke.endRow() != null + && ke.prevEndRow().compareTo(nke.endRow()) >= 0; + } + + private static Text rowAfterPrevRow(KeyExtent nke) { + Text row = new Text(nke.prevEndRow()); + row.append(new byte[] {0}, 0, 1); + return row; + } + + private CachedTablet findTabletInCache(Text row) { + + Entry<Text,CachedTablet> entry = metaCache.ceilingEntry(row); + + if (entry != null) { + KeyExtent ke = entry.getValue().getExtent(); + if (ke.prevEndRow() == null || ke.prevEndRow().compareTo(row) < 0) { + return entry.getValue(); + } + } + return null; + } + + /** + * @param cacheCutoffTimer Tablets w/o locations are cached. When LocationNeed is REQUIRED, this + * Timer value is used to determine if cached entries w/o a location should be used or of + * we should instead ignore them and reread the tablet information from the metadata table. + */ + protected CachedTablet _findTablet(ClientContext context, Text row, boolean skipRow, + LockCheckerSession lcSession, LocationNeed locationNeed, Timer cacheCutoffTimer) + throws AccumuloException, AccumuloSecurityException, TableNotFoundException, + InvalidTabletHostingRequestException { + if (skipRow) { + row = new Text(row); + row.append(new byte[] {0}, 0, 1); + } + + CachedTablet tl = lcSession.checkLock(findTabletInCache(row)); + + if (tl == null || (locationNeed == LocationNeed.REQUIRED && tl.getTserverLocation().isEmpty() + && cacheCutoffTimer.startedAfter(tl.getCreationTimer()))) { + + // not in cache OR the cutoff timer was started after when the cached entry timer was started, + // so obtain info from metadata table + tl = lookupTabletLocationAndCheckLock(context, row, lcSession, tl); + + } + + return tl; + } + + private CachedTablet lookupTabletLocationAndCheckLock(ClientContext context, Text row, + LockCheckerSession lcSession, CachedTablet before) throws AccumuloException, + AccumuloSecurityException, TableNotFoundException, InvalidTabletHostingRequestException { + lookupTablet(context, row, lcSession, before); + return lcSession.checkLock(findTabletInCache(row)); + } + + static void addRange(Map<String,Map<KeyExtent,List<Range>>> binnedRanges, CachedTablet ct, + Range range) { + binnedRanges.computeIfAbsent(ct.getTserverLocation().orElseThrow(), k -> new HashMap<>()) + .computeIfAbsent(ct.getExtent(), k -> new ArrayList<>()).add(range); + } +} diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java index 21cec3a8a1,392ba64bad..7b11472439 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java @@@ -85,11 -89,11 +85,11 @@@ public class NamespaceOperationsImpl ex timer = Timer.startNew(); } - TreeSet<String> namespaces = new TreeSet<>(Namespaces.getNameToIdMap(context).keySet()); + var namespaces = new TreeSet<>(context.getNamespaceMapping().getIdToNameMap().values()); if (timer != null) { - log.trace("tid={} Fetched {} namespaces in {}", Thread.currentThread().getId(), - namespaces.size(), String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); + log.trace("Fetched {} namespaces in {}", namespaces.size(), + String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); } return namespaces; @@@ -107,16 -110,10 +106,16 @@@ timer = Timer.startNew(); } - boolean exists = Namespaces.namespaceNameExists(context, namespace); + boolean exists = false; + try { + context.getNamespaceId(namespace); + exists = true; + } catch (NamespaceNotFoundException e) { + /* ignore */ + } if (timer != null) { - log.trace("tid={} Checked existence of {} in {}", Thread.currentThread().getId(), exists, + log.trace("Checked existence of {} in {}", exists, String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); } diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java index 4eac34a7bb,0000000000..f2197870aa mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java @@@ -1,165 -1,0 +1,164 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.clientImpl; + +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; + +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.clientImpl.ClientTabletCacheImpl.TabletServerLockChecker; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; +import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType; +import org.apache.accumulo.core.util.Timer; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides ability to get the location of the root tablet from a zoocache. This cache + * implementation does not actually do any caching on its own and soley relies on zoocache. One nice + * feature of using zoo cache is that if the location changes in zookeeper, it will eventually be + * updated by zookeeper watchers in zoocache. Therefore, the invalidation functions are + * intentionally no-ops and rely on the zookeeper watcher to keep things up to date. + * + * <p> + * This code is relying on the assumption that if two client objects are created for the same + * accumulo instance in the same process that both will have the same zoocache. This assumption + * means there is only a single zookeeper watch per process per accumulo instance. This assumptions + * leads to efficiencies at the cluster level by reduce the total number of zookeeper watches. + * </p> + */ +public class RootClientTabletCache extends ClientTabletCache { + + private final TabletServerLockChecker lockChecker; + + RootClientTabletCache(TabletServerLockChecker lockChecker) { + this.lockChecker = lockChecker; + } + + @Override + public <T extends Mutation> void binMutations(ClientContext context, List<T> mutations, + Map<String,TabletServerMutations<T>> binnedMutations, List<T> failures) { + CachedTablet rootCachedTablet = getRootTabletLocation(context); + if (rootCachedTablet != null && rootCachedTablet.getTserverLocation().isPresent()) { + var tsm = new TabletServerMutations<T>(rootCachedTablet.getTserverSession().orElseThrow()); + for (T mutation : mutations) { + tsm.addMutation(RootTable.EXTENT, mutation); + } + binnedMutations.put(rootCachedTablet.getTserverLocation().orElseThrow(), tsm); + } else { + failures.addAll(mutations); + } + } + + @Override + public List<Range> findTablets(ClientContext context, List<Range> ranges, + BiConsumer<CachedTablet,Range> rangeConsumer, LocationNeed locationNeed) { + + CachedTablet rootCachedTablet = getRootTabletLocation(context); + + if (rootCachedTablet.getTserverLocation().isEmpty() && locationNeed == LocationNeed.REQUIRED) { + // there is no location and one is required so return all ranges as failures + return ranges; + } else { + for (Range range : ranges) { + rangeConsumer.accept(rootCachedTablet, range); + } + return Collections.emptyList(); + } + } + + @Override + public void invalidateCache(KeyExtent failedExtent) { + // no-op see class level javadoc + } + + @Override + public void invalidateCache(Collection<KeyExtent> keySet) { + // no-op see class level javadoc + } + + @Override + public void invalidateCache() { + // no-op see class level javadoc + } + + protected CachedTablet getRootTabletLocation(ClientContext context) { + Logger log = LoggerFactory.getLogger(this.getClass()); + + Timer timer = null; + + if (log.isTraceEnabled()) { - log.trace("tid={} Looking up root tablet location in zookeeper.", - Thread.currentThread().getId()); ++ log.trace("Looking up root tablet location in zookeeper."); + timer = Timer.startNew(); + } + + var zooCache = context.getZooCache(); + Location loc = new RootTabletMetadata(new String(zooCache.get(RootTable.ZROOT_TABLET), UTF_8)) + .toTabletMetadata().getLocation(); + + if (timer != null) { - log.trace("tid={} Found root tablet at {} in {}", Thread.currentThread().getId(), loc, ++ log.trace("Found root tablet at {} in {}", loc, + String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); + } + + if (loc == null || loc.getType() != LocationType.CURRENT) { + return new CachedTablet(RootTable.EXTENT, Optional.empty(), Optional.empty(), + TabletAvailability.HOSTED, false); + } + + String server = loc.getHostPort(); + + if (lockChecker.isLockHeld(server, loc.getSession())) { + return new CachedTablet(RootTable.EXTENT, server, loc.getSession(), TabletAvailability.HOSTED, + false); + } else { + return new CachedTablet(RootTable.EXTENT, Optional.empty(), Optional.empty(), + TabletAvailability.HOSTED, false); + } + } + + @Override + public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow, + LocationNeed locationNeed, int hostAheadCount, Range hostAheadRange) { + + CachedTablet cachedTablet = getRootTabletLocation(context); + + // Always retry when finding the root tablet + while (cachedTablet.getTserverLocation().isEmpty() && locationNeed == LocationNeed.REQUIRED) { + sleepUninterruptibly(500, MILLISECONDS); + cachedTablet = getRootTabletLocation(context); + } + + return cachedTablet; + } +} diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java index 3a9d428649,38732fddf7..5680fa4f21 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java @@@ -201,11 -184,11 +201,11 @@@ public class TableOperationsImpl extend timer = Timer.startNew(); } - TreeSet<String> tableNames = new TreeSet<>(context.getTableNameToIdMap().keySet()); + var tableNames = new TreeSet<>(context.createQualifiedTableNameToIdMap().keySet()); if (timer != null) { - log.trace("tid={} Fetched {} table names in {}", Thread.currentThread().getId(), - tableNames.size(), String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); + log.trace("Fetched {} table names in {}", tableNames.size(), + String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); } return tableNames; @@@ -226,16 -209,10 +226,16 @@@ timer = Timer.startNew(); } - boolean exists = context.getTableNameToIdMap().containsKey(tableName); + boolean exists = false; + try { + context.getTableId(tableName); + exists = true; + } catch (TableNotFoundException e) { + /* ignore */ + } if (timer != null) { - log.trace("tid={} Checked existence of {} in {}", Thread.currentThread().getId(), exists, + log.trace("Checked existence of {} in {}", exists, String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); } diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java index 4e07195355,59871e9917..b3f660ac2f --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java @@@ -866,18 -683,18 +866,18 @@@ public final class ThriftScanner scanState.scanID = null; } - scanState.prevLoc = loc; + scanState.prevLoc = addr; if (scanState.scanID == null) { - Thread.currentThread().setName("Starting scan tserver=" + loc.tablet_location + " tableId=" - + loc.tablet_extent.tableId()); + Thread.currentThread().setName("Starting scan tserver=" + addr.serverAddress + " tableId=" + + addr.getExtent().tableId()); if (log.isTraceEnabled()) { - String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" - + loc.tablet_extent + " range=" + scanState.range + " ssil=" + String msg = "Starting scan server=" + addr.serverAddress + " type=" + addr.serverType + + " tablet=" + addr.getExtent() + " range=" + scanState.range + " ssil=" + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions + " context=" + scanState.classLoaderContext; - log.trace("tid={} {}", Thread.currentThread().getId(), msg); + log.trace("{}", msg); timer = Timer.startNew(); } diff --cc core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java index 0ddc12701c,023ec1b844..f37818866c --- a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java @@@ -83,9 -87,8 +83,9 @@@ public class MetadataCachedTabletObtain Timer timer = null; if (log.isTraceEnabled()) { - log.trace("tid={} Looking up in {} row={} stopRow={} extent={} tserver={}", - Thread.currentThread().getId(), src.getExtent().tableId(), TextUtil.truncate(row), - TextUtil.truncate(stopRow), src.getExtent(), src.getTserverLocation()); - log.trace("Looking up in {} row={} extent={} tserver={}", src.tablet_extent.tableId(), - TextUtil.truncate(row), src.tablet_extent, src.tablet_location); ++ log.trace("Looking up in {} row={} stopRow={} extent={} tserver={}", ++ src.getExtent().tableId(), TextUtil.truncate(row), TextUtil.truncate(stopRow), ++ src.getExtent(), src.getTserverLocation()); timer = Timer.startNew(); } @@@ -119,14 -122,13 +119,11 @@@ } if (timer != null) { - log.trace("tid={} Got {} results from {} in {}", Thread.currentThread().getId(), - results.size(), src.getExtent(), - log.trace("Got {} results from {} in {}", results.size(), src.tablet_extent, ++ log.trace("Got {} results from {} in {}", results.size(), src.getExtent(), String.format("%.3f secs", timer.elapsed(MILLISECONDS) / 1000.0)); } -- // if (log.isTraceEnabled()) log.trace("results "+results); -- - return MetadataLocationObtainer.getMetadataLocationEntries(results); + return MetadataCachedTabletObtainer.getMetadataLocationEntries(results); } catch (AccumuloServerException ase) { if (log.isTraceEnabled()) { diff --cc hadoop-mapreduce/pom.xml index 51898ea748,a690773cfe..85cb525665 --- a/hadoop-mapreduce/pom.xml +++ b/hadoop-mapreduce/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-hadoop-mapreduce</artifactId> <name>Apache Accumulo Hadoop MapReduce</name> diff --cc iterator-test-harness/pom.xml index 0265dfeb8c,8a0f881fa0..3d5eefdfe2 --- a/iterator-test-harness/pom.xml +++ b/iterator-test-harness/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-iterator-test-harness</artifactId> <name>Apache Accumulo Iterator Test Harness</name> diff --cc minicluster/pom.xml index 13ec60f7e6,d49cac87e5..f8c86b0e2f --- a/minicluster/pom.xml +++ b/minicluster/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-minicluster</artifactId> <name>Apache Accumulo MiniCluster</name> diff --cc minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java index 021766179c,aa1b9f3e26..0e178c0156 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java @@@ -227,10 -199,8 +226,9 @@@ public class MiniAccumuloClusterImpl im conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, "1"); conf.set("dfs.support.append", "true"); conf.set("dfs.datanode.synconclose", "true"); - conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission()); config.getHadoopConfOverrides().forEach((k, v) -> conf.set(k, v)); - String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath()); + String oldTestBuildData = + System.setProperty("test.build.data", dfs.toAbsolutePath().toString()); miniDFS.set(new MiniDFSCluster.Builder(conf).numDataNodes(config.getNumDataNodes()).build()); if (oldTestBuildData == null) { System.clearProperty("test.build.data"); diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index 4bfa81a03f,122d8c927d..010eba9570 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@@ -18,15 -18,15 +18,15 @@@ */ package org.apache.accumulo.monitor; +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; -import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly; +import static org.apache.accumulo.core.client.admin.servers.ServerId.Type.SCAN_SERVER; +import static org.apache.accumulo.core.client.admin.servers.ServerId.Type.TABLET_SERVER; import java.net.InetAddress; - import java.net.URL; + import java.net.URI; import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@@ -415,13 -524,14 +415,13 @@@ public class Monitor extends AbstractSe rootContext = rootContext + "/"; } try { - URL url = new URL(server.isSecure() ? "https" : "http", monitorHostAndPort.getHost(), - server.getPort(), rootContext); + var uri = new URI(server.isSecure() ? "https" : "http", null, monitorHostAndPort.getHost(), + server.getPort(), rootContext, null, null); - final String path = context.getZooKeeperRoot() + Constants.ZMONITOR_HTTP_ADDR; - final ZooReaderWriter zoo = context.getZooReaderWriter(); + final ZooReaderWriter zoo = context.getZooSession().asReaderWriter(); // Delete before we try to re-create in case the previous session hasn't yet expired - zoo.delete(path); - zoo.putEphemeralData(path, uri.toString().getBytes(UTF_8)); + zoo.delete(Constants.ZMONITOR_HTTP_ADDR); - zoo.putEphemeralData(Constants.ZMONITOR_HTTP_ADDR, url.toString().getBytes(UTF_8)); - log.info("Set monitor address in zookeeper to {}", url); ++ zoo.putEphemeralData(Constants.ZMONITOR_HTTP_ADDR, uri.toString().getBytes(UTF_8)); + log.info("Set monitor address in zookeeper to {}", uri); } catch (Exception ex) { log.error("Unable to advertise monitor HTTP address in zookeeper", ex); } diff --cc shell/pom.xml index 599443f38f,09a72616ab..b413fdcb4c --- a/shell/pom.xml +++ b/shell/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-shell</artifactId> <name>Apache Accumulo Shell</name> diff --cc start/pom.xml index 0766d612a9,ac1164293c..4592aae7c1 --- a/start/pom.xml +++ b/start/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-start</artifactId> <name>Apache Accumulo Start</name> diff --cc start/src/test/resources/log4j2-test.properties index 2282fe9d4f,3c1318b12b..2c64203f6d --- a/start/src/test/resources/log4j2-test.properties +++ b/start/src/test/resources/log4j2-test.properties @@@ -24,7 -24,10 +24,7 @@@ appender.console.type = Consol appender.console.name = STDOUT appender.console.target = SYSTEM_OUT appender.console.layout.type = PatternLayout - appender.console.layout.pattern = [%t} %-5p %c %x - %m%n + appender.console.layout.pattern = %d{ISO8601} Thread[%T;%t] [%-8c{2}] %-5p: %m%n -logger.01.name = org.apache.commons.vfs2.impl.DefaultFileSystemManager -logger.01.level = error - rootLogger.level = info rootLogger.appenderRef.console.ref = STDOUT diff --cc test/pom.xml index d28b2e0b04,c63128737c..a3c2a44bdc --- a/test/pom.xml +++ b/test/pom.xml @@@ -24,7 -24,8 +24,8 @@@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-project</artifactId> - <version>2.1.5-SNAPSHOT</version> + <version>4.0.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>accumulo-test</artifactId> <name>Apache Accumulo Testing</name> diff --cc test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java index d13d92237c,c925e089c5..0a03ed272c --- a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java +++ b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java @@@ -18,12 -18,10 +18,12 @@@ */ package org.apache.accumulo.test; -import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.accumulo.core.util.LazySingletons.GSON; import static org.junit.jupiter.api.Assertions.assertEquals; - import java.net.URL; + import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@@ -75,10 -75,10 +75,10 @@@ public class CountNameNodeOpsBulkIT ext justification = "path provided by test; url provided by test") private Map<?,?> getStats() throws Exception { String uri = getCluster().getMiniDfs().getHttpUri(0); - URL url = new URL(uri + "/jmx"); + var url = new URI(uri + "/jmx").toURL(); log.debug("Fetching web page " + url); String jsonString = FunctionalTestUtils.readWebPage(url).body(); - Map<?,?> jsonObject = new Gson().fromJson(jsonString, Map.class); + Map<?,?> jsonObject = GSON.get().fromJson(jsonString, Map.class); List<?> beans = (List<?>) jsonObject.get("beans"); for (Object bean : beans) { Map<?,?> map = (Map<?,?>) bean; diff --cc test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java index f9e007b41f,acdf93e329..10d465d098 --- a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java @@@ -23,10 -23,12 +23,10 @@@ import static org.apache.accumulo.harne import java.io.IOException; import java.net.HttpURLConnection; import java.net.Socket; - import java.net.URL; + import java.net.URI; import java.util.Collection; -import java.util.List; import java.util.Map; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.conf.Property; diff --cc test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java index a0e0f488dc,265f0832b1..24a1773dfe --- a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java @@@ -18,16 -18,32 +18,16 @@@ */ package org.apache.accumulo.test.functional; +import static org.apache.accumulo.core.util.LazySingletons.RANDOM; import static org.apache.accumulo.harness.AccumuloITBase.STANDALONE_CAPABLE_CLUSTER; import static org.apache.accumulo.harness.AccumuloITBase.SUNNY_DAY; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; - import java.net.URL; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; + import java.net.URI; import java.security.cert.X509Certificate; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.HostnameVerifier; diff --cc test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java index 642dee65fd,2eb578be53..6590194862 --- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java +++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java @@@ -75,35 -69,9 +75,34 @@@ import org.apache.accumulo.server.util. import org.apache.accumulo.server.util.UpgradeUtil; import org.apache.accumulo.server.util.ZooKeeperMain; import org.apache.accumulo.server.util.ZooZap; +import org.apache.accumulo.server.util.adminCommand.ChangeSecret; +import org.apache.accumulo.server.util.adminCommand.DeleteZooInstance; +import org.apache.accumulo.server.util.adminCommand.DumpConfig; +import org.apache.accumulo.server.util.adminCommand.Fate; +import org.apache.accumulo.server.util.adminCommand.ListInstances; +import org.apache.accumulo.server.util.adminCommand.ListVolumesUsed; +import org.apache.accumulo.server.util.adminCommand.Locks; +import org.apache.accumulo.server.util.adminCommand.PingServer; +import org.apache.accumulo.server.util.adminCommand.RestoreZookeeper; +import org.apache.accumulo.server.util.adminCommand.ServiceStatus; +import org.apache.accumulo.server.util.adminCommand.StopAll; +import org.apache.accumulo.server.util.adminCommand.StopManager; +import org.apache.accumulo.server.util.adminCommand.StopServers; +import org.apache.accumulo.server.util.adminCommand.SystemCheck; +import org.apache.accumulo.server.util.adminCommand.VerifyTabletAssignments; import org.apache.accumulo.shell.Shell; import org.apache.accumulo.start.Main; +import org.apache.accumulo.start.spi.CommandGroup; +import org.apache.accumulo.start.spi.CommandGroups; import org.apache.accumulo.start.spi.KeywordExecutable; +import org.apache.accumulo.test.RandomWriter; +import org.apache.accumulo.test.TestBinaryRows; +import org.apache.accumulo.test.TestIngest; +import org.apache.accumulo.test.TestMultiTableIngest; +import org.apache.accumulo.test.TestRandomDeletes; +import org.apache.accumulo.test.VerifyIngest; +import org.apache.accumulo.test.cli.TestCommandGroup; - import org.apache.accumulo.test.performance.scan.CollectTabletStats; +import org.apache.accumulo.tserver.ScanServer; import org.apache.accumulo.tserver.ScanServerExecutable; import org.apache.accumulo.tserver.TServerExecutable; import org.apache.accumulo.tserver.TabletServer; @@@ -179,95 -128,52 +178,93 @@@ public class KeywordStartIT */ @Test public void testExpectedClasses() { - assumeTrue(new File(System.getProperty("user.dir") + "/src").exists()); - TreeMap<String,Class<? extends KeywordExecutable>> expectSet = new TreeMap<>(); - expectSet.put("admin", Admin.class); - expectSet.put("check-compaction-config", CheckCompactionConfig.class); - expectSet.put("check-server-config", CheckServerConfig.class); - expectSet.put("check-accumulo-properties", CheckAccumuloProperties.class); - expectSet.put("compaction-coordinator", CoordinatorExecutable.class); - expectSet.put("compactor", CompactorExecutable.class); - expectSet.put("config-upgrade", ConfigPropertyUpgrader.class); - expectSet.put("convert-config", ConvertConfig.class); - expectSet.put("create-empty", CreateEmpty.class); - expectSet.put("create-token", CreateToken.class); - expectSet.put("dump-zoo", DumpZookeeper.class); - expectSet.put("ec-admin", ECAdmin.class); - expectSet.put("gc", GCExecutable.class); - expectSet.put("generate-splits", GenerateSplits.class); - expectSet.put("help", Help.class); - expectSet.put("info", Info.class); - expectSet.put("init", Initialize.class); - expectSet.put("login-info", LoginProperties.class); - expectSet.put("manager", ManagerExecutable.class); - expectSet.put("minicluster", MiniClusterExecutable.class); - expectSet.put("monitor", MonitorExecutable.class); - expectSet.put("rfile-info", PrintInfo.class); - expectSet.put("shell", Shell.class); - expectSet.put("split-large", SplitLarge.class); - expectSet.put("sserver", ScanServerExecutable.class); - expectSet.put("tserver", TServerExecutable.class); - expectSet.put("upgrade", UpgradeUtil.class); - expectSet.put("version", Version.class); - expectSet.put("wal-info", LogReader.class); - expectSet.put("zoo-info-viewer", ZooInfoViewer.class); - expectSet.put("zoo-prop-editor", ZooPropEditor.class); - expectSet.put("zoo-zap", ZooZap.class); - expectSet.put("zookeeper", ZooKeeperMain.class); + assumeTrue(Files.exists(Path.of(System.getProperty("user.dir")).resolve("src"))); + SortedSet<CommandInfo> expectSet = new TreeSet<>(); + expectSet.add(new CommandInfo(TestCommandGroup.INSTANCE, "binary-rows", TestBinaryRows.class)); + expectSet.add(new CommandInfo(CommandGroups.COMPACTION, "cancel", CancelCompaction.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "change-secret", ChangeSecret.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "check", SystemCheck.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "check-accumulo-properties", + CheckAccumuloProperties.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "check-compaction-config", + CheckCompactionConfig.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "compactor", CompactorExecutable.class)); + expectSet.add(new CommandInfo(CommandGroups.FILE, "create-empty", CreateEmpty.class)); + expectSet.add(new CommandInfo(CommandGroups.CLIENT, "create-token", CreateToken.class)); + expectSet + .add(new CommandInfo(CommandGroups.INSTANCE, "delete-instance", DeleteZooInstance.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "dump-config", DumpConfig.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "dump-zoo", DumpZookeeper.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "fate", Fate.class)); + expectSet.add(new CommandInfo(CommandGroups.TABLE, "find-online-ondemand-tablets", + ListOnlineOnDemandTablets.class)); + expectSet.add( + new CommandInfo(CommandGroups.TABLE, "find-offline-tablets", FindOfflineTablets.class)); + expectSet.add( + new CommandInfo(CommandGroups.COMPACTION, "find-tmp-files", FindCompactionTmpFiles.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "gc", GCExecutable.class)); + expectSet.add(new CommandInfo(CommandGroups.FILE, "generate-splits", GenerateSplits.class)); + expectSet.add(new CommandInfo(CommandGroups.CLIENT, "help", Help.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "info", Info.class)); + expectSet.add(new CommandInfo(TestCommandGroup.INSTANCE, "ingest", TestIngest.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "init", Initialize.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "list-instances", ListInstances.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "list-compactors", ListCompactors.class)); + expectSet.add(new CommandInfo(CommandGroups.COMPACTION, "list", ListCompactions.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "list-volumes", ListVolumesUsed.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "locks", Locks.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "login-info", LoginProperties.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "manager", ManagerExecutable.class)); + expectSet + .add(new CommandInfo(CommandGroups.PROCESS, "minicluster", MiniClusterExecutable.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "missing-files", + RemoveEntriesForMissingFiles.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "monitor", MonitorExecutable.class)); + expectSet.add(new CommandInfo(TestCommandGroup.INSTANCE, "multi-table-ingest", + TestMultiTableIngest.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "ping", PingServer.class)); + expectSet + .add(new CommandInfo(TestCommandGroup.INSTANCE, "random-deletes", TestRandomDeletes.class)); + expectSet.add(new CommandInfo(TestCommandGroup.INSTANCE, "random-writer", RandomWriter.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "remove-scan-server-references", + ScanServerMetadataEntries.class)); + expectSet + .add(new CommandInfo(CommandGroups.CONFIG, "restore-zookeeper", RestoreZookeeper.class)); + expectSet.add(new CommandInfo(CommandGroups.FILE, "rfile-info", PrintInfo.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "service-status", ServiceStatus.class)); + expectSet.add(new CommandInfo(CommandGroups.CLIENT, "shell", Shell.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "create-autocomplete-script", + ShellCompletionCommand.class)); + expectSet.add(new CommandInfo(CommandGroups.FILE, "split-large", SplitLarge.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "sserver", ScanServerExecutable.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "stop-all", StopAll.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "stop-manager", StopManager.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "stop-servers", StopServers.class)); - expectSet - .add(new CommandInfo(TestCommandGroup.INSTANCE, "tablet-stats", CollectTabletStats.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "tserver", TServerExecutable.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "upgrade", UpgradeUtil.class)); + expectSet.add(new CommandInfo(TestCommandGroup.INSTANCE, "verify-ingest", VerifyIngest.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "verify-tablet-assignments", + VerifyTabletAssignments.class)); + expectSet.add(new CommandInfo(CommandGroups.CLIENT, "version", Version.class)); + expectSet.add(new CommandInfo(CommandGroups.FILE, "wal-info", LogReader.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "zoo-info-viewer", ZooInfoViewer.class)); + expectSet.add(new CommandInfo(CommandGroups.CONFIG, "zoo-prop-editor", ZooPropEditor.class)); + expectSet.add(new CommandInfo(CommandGroups.PROCESS, "zoo-zap", ZooZap.class)); + expectSet.add(new CommandInfo(CommandGroups.INSTANCE, "zk-cli", ZooKeeperMain.class)); + + Map<CommandGroup,Map<String,KeywordExecutable>> actualExecutables = getKeywordExecutables(); + SortedSet<CommandInfo> actualSet = new TreeSet<>(); + actualExecutables.entrySet().forEach((e) -> { + e.getValue().entrySet().forEach((e2) -> { + actualSet.add(new CommandInfo(e.getKey(), e2.getKey(), e2.getValue().getClass())); + }); + }); - @SuppressWarnings("deprecation") - var masterExecutableClass = org.apache.accumulo.manager.MasterExecutable.class; - expectSet.put("master", masterExecutableClass); + Iterator<CommandInfo> expectIter = expectSet.iterator(); + Iterator<CommandInfo> actualIter = actualSet.iterator(); - Iterator<Entry<String,Class<? extends KeywordExecutable>>> expectIter = - expectSet.entrySet().iterator(); - TreeMap<String,KeywordExecutable> actualSet = new TreeMap<>(getKeywordExecutables()); - Iterator<Entry<String,KeywordExecutable>> actualIter = actualSet.entrySet().iterator(); - Entry<String,Class<? extends KeywordExecutable>> expected; - Entry<String,KeywordExecutable> actual; + CommandInfo expected; + CommandInfo actual; while (expectIter.hasNext() && actualIter.hasNext()) { expected = expectIter.next(); actual = actualIter.next();
