This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 14efea2e75 ignores failures to set future location (#4579)
14efea2e75 is described below
commit 14efea2e757ce91d0ff732c51c18a03950d9a719
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Wed May 22 17:43:46 2024 -0400
ignores failures to set future location (#4579)
When TabletGroupWatcher failed to set a future location it would
still ask the tablet server to load the tablet. The tablet server
would get the request and fail causing noise in its logs. This
change avoids uneeded work and noise in the logs.
---
.../server/manager/state/AbstractTabletStateStore.java | 7 ++++++-
.../server/manager/state/LoggingTabletStateStore.java | 13 ++++++++++---
.../accumulo/server/manager/state/TabletStateStore.java | 6 +++++-
.../accumulo/server/manager/state/ZooTabletStateStore.java | 6 ++++--
.../org/apache/accumulo/manager/TabletGroupWatcher.java | 7 ++++++-
5 files changed, 31 insertions(+), 8 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
index e6c78ace50..56f80eeae9 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
@@ -18,10 +18,12 @@
*/
package org.apache.accumulo.server.manager.state;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.TServerInstance;
@@ -79,7 +81,7 @@ public abstract class AbstractTabletStateStore implements
TabletStateStore {
}
@Override
- public void setFutureLocations(Collection<Assignment> assignments)
+ public Set<KeyExtent> setFutureLocations(Collection<Assignment> assignments)
throws DistributedStoreException {
try (var tabletsMutator = ample.conditionallyMutateTablets()) {
for (Assignment assignment : assignments) {
@@ -94,14 +96,17 @@ public abstract class AbstractTabletStateStore implements
TabletStateStore {
}
Map<KeyExtent,ConditionalResult> results = tabletsMutator.process();
+ List<KeyExtent> failed = new ArrayList<>();
for (Entry<KeyExtent,ConditionalResult> entry : results.entrySet()) {
if (entry.getValue().getStatus() != Status.ACCEPTED) {
LOG.debug("Likely concurrent FATE operation prevented setting future
location for {}, "
+ "Manager will retry soon.", entry.getKey());
+ failed.add(entry.getKey());
}
}
+ return Set.copyOf(failed);
} catch (RuntimeException ex) {
throw new DistributedStoreException(ex);
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index ec93787abb..0c60389848 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@ -21,8 +21,10 @@ package org.apache.accumulo.server.manager.state;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.manager.state.TabletManagement;
import org.apache.accumulo.core.metadata.TServerInstance;
@@ -61,10 +63,15 @@ class LoggingTabletStateStore implements TabletStateStore {
}
@Override
- public void setFutureLocations(Collection<Assignment> assignments)
+ public Set<KeyExtent> setFutureLocations(Collection<Assignment> assignments)
throws DistributedStoreException {
- wrapped.setFutureLocations(assignments);
- assignments.forEach(assignment -> TabletLogger.assigned(assignment.tablet,
assignment.server));
+ var failures = wrapped.setFutureLocations(assignments);
+ assignments.forEach(assignment -> {
+ if (!failures.contains(assignment.tablet)) {
+ TabletLogger.assigned(assignment.tablet, assignment.server);
+ }
+ });
+ return failures;
}
@Override
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
index 602409125e..c4f3910d51 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -65,8 +66,11 @@ public interface TabletStateStore {
/**
* Store the assigned locations in the data store.
+ *
+ * @return the failures, the extents that a future location could not be set
for
*/
- void setFutureLocations(Collection<Assignment> assignments) throws
DistributedStoreException;
+ Set<KeyExtent> setFutureLocations(Collection<Assignment> assignments)
+ throws DistributedStoreException;
/**
* Tablet servers will update the data store with the location when they
bring the tablet online
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
index 504d9e5589..ce5d77b08e 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
@@ -28,11 +28,13 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator;
import org.apache.accumulo.core.manager.state.TabletManagement;
@@ -154,10 +156,10 @@ class ZooTabletStateStore extends
AbstractTabletStateStore implements TabletStat
}
@Override
- public void setFutureLocations(Collection<Assignment> assignments)
+ public Set<KeyExtent> setFutureLocations(Collection<Assignment> assignments)
throws DistributedStoreException {
validateAssignments(assignments);
- super.setFutureLocations(assignments);
+ return super.setFutureLocations(assignments);
}
@Override
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index e15b289118..1b41145fa8 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -1026,12 +1026,17 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
flushLock.unlock();
}
+ Set<KeyExtent> failedFuture = Set.of();
if (!tLists.assignments.isEmpty()) {
Manager.log.info(String.format("Assigning %d tablets",
tLists.assignments.size()));
- store.setFutureLocations(tLists.assignments);
+ failedFuture = store.setFutureLocations(tLists.assignments);
}
tLists.assignments.addAll(tLists.assigned);
for (Assignment a : tLists.assignments) {
+ if (failedFuture.contains(a.tablet)) {
+ // do not ask a tserver to load a tablet where the future location
could not be set
+ continue;
+ }
try {
TServerConnection client = manager.tserverSet.getConnection(a.server);
if (client != null) {
