This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 00b7d934de shows more info for bulk imports in monitor (#6213)
00b7d934de is described below
commit 00b7d934de58e9647bfea308a2a8b18422b0091d
Author: Keith Turner <[email protected]>
AuthorDate: Thu Apr 2 12:26:15 2026 -0700
shows more info for bulk imports in monitor (#6213)
Made the following changes
* Show table id and fate id for each bulk import in the monitor.
* Memoize gathering bulk import data to avoid the monitor scanning the
fate table too much.
* Removed some vestigial bulkv1 code from the monitor.
fixes #6150
---
.../java/org/apache/accumulo/monitor/Monitor.java | 20 +++++++++
.../rest/bulkImports/BulkImportInformation.java | 17 +++++---
.../rest/bulkImports/BulkImportResource.java | 9 +---
.../TabletServerBulkImportInformation.java | 50 ----------------------
.../accumulo/monitor/resources/js/bulkImport.js | 47 +++++---------------
.../accumulo/monitor/templates/bulkImport.ftl | 2 +
.../accumulo/test/MultipleManagerFateIT.java | 2 +
7 files changed, 47 insertions(+), 100 deletions(-)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 9e53765c02..9e1546c15d 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -74,9 +74,12 @@ import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.monitor.next.InformationFetcher;
+import org.apache.accumulo.monitor.rest.bulkImports.BulkImport;
+import org.apache.accumulo.monitor.rest.bulkImports.BulkImportInformation;
import org.apache.accumulo.server.AbstractServer;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.util.TableInfoUtil;
+import org.apache.accumulo.server.util.bulkCommand.ListBulk;
import org.apache.zookeeper.KeeperException;
import org.eclipse.jetty.ee10.servlet.ResourceServlet;
import org.eclipse.jetty.ee10.servlet.ServletHolder;
@@ -552,6 +555,9 @@ public class Monitor extends AbstractServer implements
Connection.Listener {
private final Supplier<Map<HostAndPort,CompactionStats>> compactionsSupplier
=
Suppliers.memoizeWithExpiration(this::fetchCompactions,
expirationTimeMinutes, MINUTES);
+ private final Supplier<BulkImport> bulkImportSupplier =
+ Suppliers.memoizeWithExpiration(this::computeBulkImports,
expirationTimeMinutes, MINUTES);
+
/**
* @return active tablet server scans. Values are cached and refresh after
* {@link #expirationTimeMinutes}.
@@ -575,6 +581,20 @@ public class Monitor extends AbstractServer implements
Connection.Listener {
return compactionsSupplier.get();
}
+ private BulkImport computeBulkImports() {
+ BulkImport bulkImport = new BulkImport();
+ ListBulk.list(getContext(), bulkStatus -> {
+ bulkImport.addBulkImport(
+ new BulkImportInformation(bulkStatus.sourceDir(),
bulkStatus.lastUpdate().toEpochMilli(),
+ bulkStatus.state(), bulkStatus.tableId(), bulkStatus.fateId()));
+ });
+ return bulkImport;
+ }
+
+ public BulkImport getBulkImports() {
+ return bulkImportSupplier.get();
+ }
+
private Map<HostAndPort,ScanStats> fetchScans(Collection<ServerId> servers) {
ServerContext context = getContext();
Map<HostAndPort,ScanStats> scans = new HashMap<>();
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java
index 0919751070..cd84b0b34b 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportInformation.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.monitor.rest.bulkImports;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.server.util.bulkCommand.ListBulk;
/**
@@ -28,11 +30,11 @@ import org.apache.accumulo.server.util.bulkCommand.ListBulk;
public class BulkImportInformation {
// Variable names become JSON key
- public String filename;
- public long age;
- public ListBulk.BulkState state;
-
- public BulkImportInformation() {}
+ public final String filename;
+ public final long age;
+ public final ListBulk.BulkState state;
+ public final String tableId;
+ public final String fateId;
/**
* Creates new bulk import object
@@ -41,9 +43,12 @@ public class BulkImportInformation {
* @param age age of the bulk import
* @param state state of the bulk import
*/
- public BulkImportInformation(String filename, long age, ListBulk.BulkState
state) {
+ public BulkImportInformation(String filename, long age, ListBulk.BulkState
state, TableId tableId,
+ FateId fateId) {
this.filename = filename;
this.age = age;
this.state = state;
+ this.tableId = tableId.canonical();
+ this.fateId = fateId.canonical();
}
}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
index 7eea08ed31..2a15b1ec75 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
@@ -25,7 +25,6 @@ import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import org.apache.accumulo.monitor.Monitor;
-import org.apache.accumulo.server.util.bulkCommand.ListBulk;
/**
* The BulkImportResource is responsible for obtaining the information of the
bulk import, and
@@ -47,12 +46,6 @@ public class BulkImportResource {
*/
@GET
public BulkImport getTables() {
- BulkImport bulkImport = new BulkImport();
- ListBulk.list(monitor.getContext(), bulkStatus -> {
- bulkImport.addBulkImport(new
BulkImportInformation(bulkStatus.sourceDir(),
- bulkStatus.lastUpdate().toEpochMilli(), bulkStatus.state()));
- });
-
- return bulkImport;
+ return monitor.getBulkImports();
}
}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/TabletServerBulkImportInformation.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/TabletServerBulkImportInformation.java
deleted file mode 100644
index 168065dc5c..0000000000
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/TabletServerBulkImportInformation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.monitor.rest.bulkImports;
-
-import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
-
-/**
- * Stores tserver bulk import information
- *
- * @since 2.0.0
- */
-public class TabletServerBulkImportInformation {
-
- // Variable names become JSON key
- public String server;
- public int importSize;
- public long oldestAge;
-
- public TabletServerBulkImportInformation() {}
-
- /**
- * Creates a new tserver bulk import object
- *
- * @param server server name
- * @param importSize import size
- * @param oldestAge tserver bulk import age
- */
- public TabletServerBulkImportInformation(TabletServerStatus server, int
importSize,
- long oldestAge) {
- this.server = server.getName();
- this.importSize = importSize;
- this.oldestAge = oldestAge;
- }
-}
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
index 735f4595ad..9b22b6781a 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
@@ -51,12 +51,20 @@ $(function () {
"stateSave": true,
"autoWidth": false,
"columns": [{
+ "data": "tableId",
+ "width": "5%"
+ },
+ {
+ "data": "fateId",
+ "width": "25%"
+ },
+ {
"data": "filename",
- "width": "40%"
+ "width": "35%"
},
{
"data": "age",
- "width": "45%",
+ "width": "25%",
"render": function (data, type) {
var age = Number(data);
if (type === 'display') {
@@ -67,40 +75,7 @@ $(function () {
},
{
"data": "state",
- "width": "15%"
- }
- ]
- });
-
- // Generates the bulkPerServerTable DataTable
- bulkPerServerTable = $('#bulkPerServerTable').DataTable({
- "ajax": {
- "url": url,
- "dataSrc": "tabletServerBulkImport"
- },
- "stateSave": true,
- "columns": [{
- "data": "server",
- "type": "html",
- "render": function (data, type) {
- if (type === 'display') {
- data = `<a href="tservers?s=${data}">${data}</a>`;
- }
- return data;
- }
- },
- {
- "data": "importSize"
- },
- {
- "data": "oldestAge",
- "render": function (data, type) {
- var age = Number(data);
- if (type === 'display') {
- return age > 0 ? new Date(age) : "-";
- }
- return age > 0 ? age : 0;
- }
+ "width": "10%"
}
]
});
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/bulkImport.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/bulkImport.ftl
index d172fc838a..8ae6cf9b5e 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/bulkImport.ftl
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/bulkImport.ftl
@@ -29,6 +29,8 @@
<caption><span class="table-caption">Bulk Imports</span><br
/></caption>
<thead>
<tr>
+ <th>Table ID </th>
+ <th>Fate ID </th>
<th>Directory </th>
<th title="The age of the import.">Age </th>
<th title="The current state of the bulk
import">State </th>
diff --git
a/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java
b/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java
index d0d8bdb5ed..5e825efbac 100644
--- a/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java
@@ -231,6 +231,8 @@ public class MultipleManagerFateIT extends
ConfigurableMacBase {
log.debug("Deleted lock of primary manager");
waitToSeeManagers(ctx, 2, store, true);
+ getCluster().getProcesses();
+
stop.set(true);
// Wait for the background operations to complete and ensure that none
had errors. Managers
// stoppping/starting should not cause any problems for Accumulo API
operations.
