This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d8a5d26884 Added monitor activity page for Fate transaction details
(#6371)
d8a5d26884 is described below
commit d8a5d26884b4c5284a3248d32306496c01beb897
Author: Dave Marion <[email protected]>
AuthorDate: Wed May 20 13:19:04 2026 -0400
Added monitor activity page for Fate transaction details (#6371)
Co-authored-by: Dom G. <[email protected]>
---
.../apache/accumulo/monitor/next/Endpoints.java | 9 ++
.../accumulo/monitor/next/InformationFetcher.java | 96 +++++++++++++++-
.../accumulo/monitor/next/SystemInformation.java | 28 +++++
.../org/apache/accumulo/monitor/view/WebViews.java | 18 +++
.../apache/accumulo/monitor/resources/js/fate.js | 125 +++++++++++++++++++++
.../accumulo/monitor/resources/js/functions.js | 9 ++
.../org/apache/accumulo/monitor/templates/fate.ftl | 30 +++++
.../apache/accumulo/monitor/templates/navbar.ftl | 1 +
8 files changed, 314 insertions(+), 2 deletions(-)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java
index f5719637b2..d48173f821 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java
@@ -58,6 +58,7 @@ import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.monitor.next.InformationFetcher.InstanceSummary;
import
org.apache.accumulo.monitor.next.SystemInformation.CompactionGroupSummary;
import
org.apache.accumulo.monitor.next.SystemInformation.CompactionTableSummary;
+import org.apache.accumulo.monitor.next.SystemInformation.FateTransaction;
import org.apache.accumulo.monitor.next.SystemInformation.MessageCategory;
import org.apache.accumulo.monitor.next.SystemInformation.MessagePriority;
import org.apache.accumulo.monitor.next.SystemInformation.RecoveryInformation;
@@ -387,6 +388,14 @@ public class Endpoints {
return new CompactorsSummary(summary.getCompactorServers(),
summary.getTimestamp());
}
+ @GET
+ @Path("fate")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Description("Returns a list of fate transaction details")
+ public List<FateTransaction> getFateTransactions() {
+ return
monitor.getInformationFetcher().getSummaryForEndpoint().getFateTransactions();
+ }
+
@GET
@Path("tables")
@Produces(MediaType.APPLICATION_JSON)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
index 5e0aec350b..0f04f818ab 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CancellationException;
@@ -53,6 +54,12 @@ import
org.apache.accumulo.core.client.admin.servers.ServerId.Type;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.fate.AdminUtil;
+import org.apache.accumulo.core.fate.AdminUtil.FateStatus;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.MetaFateStore;
import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector;
import org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate;
import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath;
@@ -74,6 +81,8 @@ import
org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.compaction.CompactionPluginUtils;
+import org.apache.accumulo.server.util.adminCommand.Fate;
+import org.apache.zookeeper.KeeperException;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.eclipse.jetty.util.NanoTime;
import org.slf4j.Logger;
@@ -173,7 +182,7 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
}
enum UpdateType {
- COMPACTION, COMPACTION_RGS, METRIC, TABLE;
+ COMPACTION, COMPACTION_RGS, FATE, METRIC, TABLE;
}
interface UpdateTask<T extends Object> extends Runnable,
Comparable<UpdateTask<T>> {
@@ -412,6 +421,71 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
}
}
+ class FateTransactionFetcher implements UpdateTask<Void> {
+
+ private final SystemInformation summary;
+
+ public FateTransactionFetcher(SystemInformation summary) {
+ this.summary = summary;
+ }
+
+ @Override
+ public void run() {
+ try {
+ AdminUtil<Fate> admin = new AdminUtil<>();
+ var zTableLocksPath = ctx.getServerPaths().createTableLocksPath();
+ var zk = ctx.getZooSession();
+ FateStatus status = admin.getStatus(stores, zk, zTableLocksPath, null,
null, null);
+ summary.processFateTransactions(status.getTransactions());
+ } catch (KeeperException | InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Objects.hash(getType());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ FateTransactionFetcher other = (FateTransactionFetcher) obj;
+ return Objects.equals(getType(), other.getType());
+ }
+
+ @Override
+ public int compareTo(UpdateTask<Void> other) {
+ return this.getType().compareTo(other.getType());
+ }
+
+ @Override
+ public UpdateType getType() {
+ return UpdateType.FATE;
+ }
+
+ @Override
+ public Void getResource() {
+ return null;
+ }
+
+ @Override
+ public String getFailureMessage() {
+ return "Error fetching fate transaction details";
+ }
+ }
+
class ConfiguredCompactionResourceGroupFetcher implements UpdateTask<Void> {
private final SystemInformation summary;
@@ -486,6 +560,9 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
private final Cache<ServerId,MetricResponse> allMetrics;
private final Cache<ServerId,Boolean> retainedProblemServers;
private final AtomicReference<SystemInformation> summaryRef = new
AtomicReference<>();
+ private final ReadOnlyFateStore<Fate> readOnlyMFS;
+ private final ReadOnlyFateStore<Fate> readOnlyUFS;
+ private final Map<FateInstanceType,ReadOnlyFateStore<Fate>> stores;
private final TabletMetadataFilter noLocation = new
NoCurrentLocationFilter();
public InformationFetcher(ServerContext ctx, Supplier<Long> connectionCount)
{
@@ -495,6 +572,13 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
.expireAfterWrite(Duration.ofMinutes(10)).evictionListener(this::onRemoval).build();
this.retainedProblemServers = Caffeine.newBuilder().executor(pool)
.scheduler(Scheduler.systemScheduler()).expireAfterWrite(Duration.ofMinutes(10)).build();
+ try {
+ this.readOnlyMFS = new MetaFateStore<>(ctx.getZooSession(), null, null);
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException("Exception creating MetaFateStore", e);
+ }
+ this.readOnlyUFS = new UserFateStore<>(ctx, SystemTables.FATE.tableName(),
null, null);
+ this.stores = Map.of(FateInstanceType.META, readOnlyMFS,
FateInstanceType.USER, readOnlyUFS);
}
public void newConnectionEvent() {
@@ -659,9 +743,16 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
final UpdateTasks futures = new UpdateTasks();
final SystemInformation summary = new SystemInformation(allMetrics,
this.ctx);
+
+ // Fetch set of registered compactors
Set<ServerId> compactors =
this.ctx.instanceOperations().getServers(Type.COMPACTOR);
summary.processExternalCompactionInventory(compactors);
+ // Fetch Fate transaction information
+ FateTransactionFetcher fateFetcher = new FateTransactionFetcher(summary);
+ Future<?> fff = this.pool.submit(fateFetcher);
+ futures.add(new UpdateTaskFuture(fff, fateFetcher));
+
// Fetch metrics from the other server processes. This
// makes an RPC call to AbstractServer.getMetrics
for (ServerId.Type type : ServerId.Type.values()) {
@@ -674,7 +765,6 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
futures.add(new UpdateTaskFuture(mff, mf));
}
}
- ThreadPools.resizePool(pool, () -> Math.max(20, (futures.size() / 20)),
poolName);
// Fetch external compaction information from the Compactors
RunningCompactionFetcher rcf = new RunningCompactionFetcher(summary,
pool);
@@ -692,6 +782,8 @@ public class InformationFetcher implements
RemovalListener<ServerId,MetricRespon
Future<?> f = this.pool.submit(r);
futures.add(new UpdateTaskFuture(f, r));
+ ThreadPools.resizePool(pool, () -> Math.max(20, (futures.size() / 20)),
poolName);
+
final long monitorFetchTimeout =
ctx.getConfiguration().getTimeInMillis(Property.MONITOR_FETCH_TIMEOUT);
final long allFuturesAdded = NanoTime.now();
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
index 7700e1d6f6..e27991799b 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java
@@ -67,6 +67,10 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.TabletId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.TabletIdImpl;
+import org.apache.accumulo.core.fate.AdminUtil.TransactionStatus;
+import org.apache.accumulo.core.fate.Fate.FateOperation;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus;
import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector;
import org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate;
import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath;
@@ -457,6 +461,14 @@ public class SystemInformation {
}
}
+ public enum LockRangeType {
+ FULL, PARTIAL;
+ }
+
+ public record FateTransaction(FateInstanceType type, FateOperation op,
String id, TStatus status,
+ long created, List<String> heldLocks, List<String> waitingLocks,
LockRangeType lockRange) {
+ }
+
private static final Logger LOG =
LoggerFactory.getLogger(SystemInformation.class);
private final DistributionStatisticConfig DSC =
@@ -526,6 +538,8 @@ public class SystemInformation {
private final Set<String> configuredCompactionResourceGroups =
ConcurrentHashMap.newKeySet();
+ private final List<FateTransaction> fateTransactions = new ArrayList<>();
+
private final AtomicLong timestamp = new AtomicLong(0);
private final EnumMap<ServerId.Type,Status> componentStatuses =
new EnumMap<>(ServerId.Type.class);
@@ -575,6 +589,7 @@ public class SystemInformation {
componentStatuses.clear();
managerGoalState = null;
serverMetricsView.clear();
+ fateTransactions.clear();
messageCounts.clear();
}
@@ -909,6 +924,15 @@ public class SystemInformation {
}
}
+ public void processFateTransactions(List<TransactionStatus> transactions) {
+ transactions.forEach(t -> {
+ fateTransactions
+ .add(new FateTransaction(t.getInstanceType(), t.getFateOp(),
t.getFateId().getTxUUIDStr(),
+ t.getStatus(), t.getTimeCreated(), t.getHeldLocks(),
t.getWaitingLocks(),
+ t.getLockRange().isInfinite() ? LockRangeType.FULL :
LockRangeType.PARTIAL));
+ });
+ }
+
public void processError(ServerId server) {
problemHosts.add(server);
}
@@ -1375,6 +1399,10 @@ public class SystemInformation {
return this.messages;
}
+ public List<FateTransaction> getFateTransactions() {
+ return this.fateTransactions;
+ }
+
public Map<MessagePriority,AtomicLong> getMessageCounts() {
return this.messageCounts;
}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
index f373ea2463..4f299706ef 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
@@ -346,6 +346,24 @@ public class WebViews {
return model;
}
+ /**
+ * Returns the Fate template
+ *
+ * @return Fate model
+ */
+ @GET
+ @Path("fate")
+ @Template(name = "/default.ftl")
+ public Map<String,Object> getFate() {
+
+ Map<String,Object> model = getModel();
+ model.put("title", "Fate Transaction Details");
+ model.put("template", "fate.ftl");
+ model.put("js", "fate.js");
+
+ return model;
+ }
+
/**
* Returns the garbage collector template
*
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js
new file mode 100644
index 0000000000..83ca0c69a1
--- /dev/null
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+"use strict";
+
+const fateHtmlTable = '#fateTable';
+
+var dataTableRef;
+
+function getTableData() {
+ return getStoredArray(FATE);
+}
+
+/**
+ * Renders array as comma separated list or a dash if the list is empty
+ */
+function renderListOrDash(data, type) {
+ if (Array.isArray(data)) {
+ if (data.length === 0) {
+ return type === 'display' ? '—' : '';
+ }
+ return data.join(', ');
+ }
+ if (data === null || data === undefined || data === '') {
+ return type === 'display' ? '—' : '';
+ }
+ return data;
+}
+
+function createDataTable() {
+ $(fateHtmlTable).find('thead').remove();
+ $(fateHtmlTable).find('tbody').remove();
+ dataTableRef = $(fateHtmlTable).DataTable({
+ "autoWidth": false,
+ "ajax": function (data, callback) {
+ callback({
+ data: getTableData()
+ });
+ },
+ "stateSave": true,
+ "colReorder": true,
+ "columnDefs": [{
+ targets: '_all',
+ defaultContent: '-'
+ }],
+ "columns": [{
+ "data": "type",
+ "title": "Type"
+ },
+ {
+ "data": "op",
+ "title": "Operation"
+ },
+ {
+ "data": "id",
+ "title": "Id"
+ },
+ {
+ "data": "status",
+ "title": "State"
+ },
+ {
+ "data": "created",
+ "title": "Created",
+ "render": function (data, type, row) {
+ if (type === 'display') data = dateFormat(data);
+ return data;
+ }
+ },
+ {
+ "data": "created",
+ "title": "Age",
+ "render": function (data, type, row) {
+ var dur = Date.now() - data;
+ if (type === 'display') dur = timeDuration(dur);
+ return dur;
+ }
+ },
+ {
+ "data": "heldLocks",
+ "title": "Locks Held",
+ "render": renderListOrDash
+ },
+ {
+ "data": "waitingLocks",
+ "title": "Locks Waiting On",
+ "render": renderListOrDash
+ },
+ {
+ "data": "lockRange",
+ "title": "Lock Range Type"
+ }
+ ]
+ });
+}
+
+function refresh() {
+ return getFate().then(function () {
+ if (dataTableRef) {
+ ajaxReloadTable(dataTableRef);
+ }
+ });
+}
+
+
+$(function () {
+ getFate().then(function () {
+ createDataTable();
+ });
+});
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js
index 15adaf0397..05b0613ee4 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js
@@ -40,6 +40,7 @@ const RUNNING_COMPACTIONS_BY_GROUP =
'runningCompactionsByGroup';
const AUTO_REFRESH_KEY = 'auto-refresh';
const MESSAGE_CATEGORIES = 'messageCategories';
const MESSAGES = 'messages';
+const FATE = 'fate';
const MESSAGE_COUNTS = 'messageCounts'
const RECOVERY = 'recovery';
@@ -715,6 +716,14 @@ function getDeployment() {
return getJSONForTable(REST_V2_PREFIX + '/deployment', 'deployment');
}
+/**
+ * REST GET call for /fate,
+ * stores it on a sessionStorage variable
+ */
+function getFate() {
+ return getJSONForTable(REST_V2_PREFIX + '/fate', FATE);
+}
+
function getServerProcessView(table, storageKey) {
var url = REST_V2_PREFIX + '/servers/view;table=' + table;
return getJSONForTable(url, storageKey);
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl
new file mode 100644
index 0000000000..b460c4a947
--- /dev/null
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl
@@ -0,0 +1,30 @@
+<#--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+ <div class="row">
+ <div class="col-xs-12">
+ <table id="fateTable" class="table caption-top table-bordered
table-striped table-condensed">
+ <caption><span class="table-caption">Fate Transaction
Details</span><br />
+ <span class="table-subcaption">The table contains the last known
Fate transaction status.</span><br />
+ </caption>
+ <#include "table_loading.ftl" >
+ </table>
+ </div>
+ </div>
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
index 841c3f44d0..ea3d317ff8 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
@@ -57,6 +57,7 @@
<li><a class="link-body-emphasis dropdown-item"
href="bulkImports">Bulk Imports</a></li>
<li><a class="link-body-emphasis dropdown-item"
href="coordinator">Compaction Overview</a></li>
<li><a class="link-body-emphasis dropdown-item"
href="ec">Compaction Details</a></li>
+ <li><a class="link-body-emphasis dropdown-item"
href="fate">Fate Tx Details</a></li>
<li><a class="link-body-emphasis dropdown-item"
href="scans">Scans</a></li>
<li><a class="link-body-emphasis dropdown-item"
href="recovery">Tablet Recoveries</a></li>
</ul>
