This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 5d06a2a635f2 CAMEL-26691: timeline - route events, circuit breakers
and log lines, and fixes from trying it on the examples (#29384)
5d06a2a635f2 is described below
commit 5d06a2a635f2e1d1783cd5264c57d0e6ca0e1600
Author: Claus Ibsen <[email protected]>
AuthorDate: Sat Nov 7 20:16:35 2026 +0100
CAMEL-26691: timeline - route events, circuit breakers and log lines, and
fixes from trying it on the examples (#29384)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/catalog/dev-consoles-openapi.json | 52 +++++++++
.../camel/catalog/dev-consoles/timeline.json | 52 +++++++++
.../org/apache/camel/dev-console/timeline.json | 52 +++++++++
.../camel/impl/console/TimelineDevConsole.java | 116 ++++++++++++++++++-
.../camel/impl/console/TimelineRecorder.java | 37 ++++++
.../camel/impl/console/TimelineDevConsoleTest.java | 83 ++++++++++++++
.../jbang/camel-tui-timeline-circuit-breaker.png | Bin 0 -> 250813 bytes
.../ROOT/pages/camel-jbang-diagnostics.adoc | 4 +
.../modules/ROOT/pages/camel-jbang-mcp.adoc | 3 +-
.../ROOT/pages/camel-jbang-tui-observe.adoc | 18 ++-
.../dsl/jbang/core/commands/ai/ToolRegistry.java | 3 +-
.../jbang/core/commands/process/ListTimeline.java | 20 ++++
.../core/commands/process/ListTimelineTest.java | 13 ++-
.../dsl/jbang/core/commands/mcp/RuntimeTools.java | 1 +
.../core/commands/tui/DataRefreshService.java | 17 +++
.../dsl/jbang/core/commands/tui/Timeline.java | 111 +++++++++++++++++-
.../jbang/core/commands/tui/TimelineExchanges.java | 20 +++-
.../dsl/jbang/core/commands/tui/TimelineLog.java | 127 +++++++++++++++++++++
.../dsl/jbang/core/commands/tui/TimelineTab.java | 90 ++++++++++++++-
.../src/main/resources/tui/help/timeline.md | 35 ++++--
.../dsl/jbang/core/commands/tui/TimelineJson.java | 15 +++
.../jbang/core/commands/tui/TimelineLogTest.java | 117 +++++++++++++++++++
.../core/commands/tui/TimelineTabRenderTest.java | 54 +++++++++
.../dsl/jbang/core/commands/tui/TimelineTest.java | 28 +++++
24 files changed, 1038 insertions(+), 30 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles-openapi.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles-openapi.json
index 39d0fc65bd30..2ee82cc0952b 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles-openapi.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles-openapi.json
@@ -9223,6 +9223,58 @@
},
"description": "The faults per second of each camel-chaos rule that
fired in the seconds (only present when one did)"
},
+
"routeEvents": {
+
"type": "array",
+
"items": {
+
"type": "object",
+
"properties": {
+
"routeId": {
+
"type": "string",
+
"description": "The route ID"
+
},
+
"event": {
+
"type": "string",
+
"description": "What happened to the route:
started, stopped, reloaded, removed or restartFailed"
+
},
+
"second": {
+
"type": "integer",
+
"description": "The epoch second it happened"
+
}
+
},
+
"required": [
+
"second"
+
]
+
},
+
"description": "What happened to the routes in the seconds: started,
stopped, reloaded, removed (only present when anything did)"
+
},
+
"circuitBreakers": {
+
"type": "array",
+
"items": {
+
"type": "object",
+
"properties": {
+
"routeId": {
+
"type": "string",
+
"description": "The route ID"
+
},
+
"id": {
+
"type": "string",
+
"description": "The id of the circuit breaker
EIP (not present for a route policy)"
+
},
+
"kind": {
+
"type": "string",
+
"description": "The kind of circuit breaker:
resilience4j, fault-tolerance or route-policy"
+
},
+
"states": {
+
"type": "array",
+
"items": {
+
"type": "integer"
+
},
+
"description": "Per second 2 when the circuit
breaker was open, 1 half open, else 0"
+
}
+
}
+
},
+
"description": "The circuit breakers that were open or half open in the
seconds (only present when one was)"
+
},
"exchanges": {
"type": "array",
"items": {
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/timeline.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/timeline.json
index fc9b5da8c4b2..3e7645e18534 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/timeline.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dev-consoles/timeline.json
@@ -218,6 +218,58 @@
},
"description": "The faults per second of each camel-chaos rule that
fired in the seconds (only present when one did)"
},
+ "routeEvents": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "routeId": {
+ "type": "string",
+ "description": "The route ID"
+ },
+ "event": {
+ "type": "string",
+ "description": "What happened to the route: started, stopped,
reloaded, removed or restartFailed"
+ },
+ "second": {
+ "type": "integer",
+ "description": "The epoch second it happened"
+ }
+ },
+ "required": [
+ "second"
+ ]
+ },
+ "description": "What happened to the routes in the seconds: started,
stopped, reloaded, removed (only present when anything did)"
+ },
+ "circuitBreakers": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "routeId": {
+ "type": "string",
+ "description": "The route ID"
+ },
+ "id": {
+ "type": "string",
+ "description": "The id of the circuit breaker EIP (not present
for a route policy)"
+ },
+ "kind": {
+ "type": "string",
+ "description": "The kind of circuit breaker: resilience4j,
fault-tolerance or route-policy"
+ },
+ "states": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ },
+ "description": "Per second 2 when the circuit breaker was open,
1 half open, else 0"
+ }
+ }
+ },
+ "description": "The circuit breakers that were open or half open in
the seconds (only present when one was)"
+ },
"exchanges": {
"type": "array",
"items": {
diff --git
a/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/timeline.json
b/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/timeline.json
index fc9b5da8c4b2..3e7645e18534 100644
---
a/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/timeline.json
+++
b/core/camel-console/src/generated/resources/META-INF/org/apache/camel/dev-console/timeline.json
@@ -218,6 +218,58 @@
},
"description": "The faults per second of each camel-chaos rule that
fired in the seconds (only present when one did)"
},
+ "routeEvents": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "routeId": {
+ "type": "string",
+ "description": "The route ID"
+ },
+ "event": {
+ "type": "string",
+ "description": "What happened to the route: started, stopped,
reloaded, removed or restartFailed"
+ },
+ "second": {
+ "type": "integer",
+ "description": "The epoch second it happened"
+ }
+ },
+ "required": [
+ "second"
+ ]
+ },
+ "description": "What happened to the routes in the seconds: started,
stopped, reloaded, removed (only present when anything did)"
+ },
+ "circuitBreakers": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "routeId": {
+ "type": "string",
+ "description": "The route ID"
+ },
+ "id": {
+ "type": "string",
+ "description": "The id of the circuit breaker EIP (not present
for a route policy)"
+ },
+ "kind": {
+ "type": "string",
+ "description": "The kind of circuit breaker: resilience4j,
fault-tolerance or route-policy"
+ },
+ "states": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ },
+ "description": "Per second 2 when the circuit breaker was open,
1 half open, else 0"
+ }
+ }
+ },
+ "description": "The circuit breakers that were open or half open in
the seconds (only present when one was)"
+ },
"exchanges": {
"type": "array",
"items": {
diff --git
a/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineDevConsole.java
b/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineDevConsole.java
index cdff0369c659..355684d9b4c2 100644
---
a/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineDevConsole.java
+++
b/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineDevConsole.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
@@ -62,6 +63,11 @@ public class TimelineDevConsole extends AbstractDevConsole {
private static final Logger LOG =
LoggerFactory.getLogger(TimelineDevConsole.class);
+ // the dev consoles with circuit breakers, and the kind of circuit breaker
each reports
+ private static final String[][] BREAKER_CONSOLES = {
+ { "resilience4j", "resilience4j" }, { "fault-tolerance",
"fault-tolerance" },
+ { "circuit-breaker", "route-policy" } };
+
/** The exchange property camel-chaos marks a faulted exchange with, with
the id of the rule. */
static final String CHAOS_RULE = "CamelChaosRule";
@@ -85,12 +91,25 @@ public class TimelineDevConsole extends AbstractDevConsole {
@Metadata(description = "Per second 1 when a camel-chaos rule had
the route down (stall or a lifecycle fault), else 0 (only present with
camel-chaos)") List<Long> down) {
}
+ public record RouteEvent(
+ @Metadata(description = "The route ID") String routeId,
+ @Metadata(description = "What happened to the route: started,
stopped, reloaded, removed or restartFailed") String event,
+ @Metadata(description = "The epoch second it happened") long
second) {
+ }
+
public record ChaosSeries(
@Metadata(description = "The id of the camel-chaos rule") String
id,
@Metadata(description = "The rule in its one-line text form")
String rule,
@Metadata(description = "The faults the rule fired per second")
List<Long> faults) {
}
+ public record CircuitBreakerSeries(
+ @Metadata(description = "The route ID") String routeId,
+ @Metadata(description = "The id of the circuit breaker EIP (not
present for a route policy)") String id,
+ @Metadata(description = "The kind of circuit breaker:
resilience4j, fault-tolerance or route-policy") String kind,
+ @Metadata(description = "Per second 2 when the circuit breaker was
open, 1 half open, else 0") List<Long> states) {
+ }
+
public record ErrorSeries(
@Metadata(description = "The route ID where the exchange failed")
String routeId,
@Metadata(description = "The exception class name") String
exception,
@@ -120,6 +139,8 @@ public class TimelineDevConsole extends AbstractDevConsole {
@Metadata(description = "The routes") List<RouteSeries> routes,
@Metadata(description = "The errors by route and exception (only
present when there are any)") List<ErrorSeries> errors,
@Metadata(description = "The faults per second of each camel-chaos
rule that fired in the seconds (only present when one did)") List<ChaosSeries>
chaos,
+ @Metadata(description = "What happened to the routes in the
seconds: started, stopped, reloaded, removed (only present when anything did)")
List<RouteEvent> routeEvents,
+ @Metadata(description = "The circuit breakers that were open or
half open in the seconds (only present when one was)")
List<CircuitBreakerSeries> circuitBreakers,
@Metadata(description = "Per second the failed, handled and
recovered exchanges (up to 10) and the 3 slowest, oldest first (only present
when asked for)") List<TimelineExchange> exchanges) {
}
@@ -194,6 +215,56 @@ public class TimelineDevConsole extends AbstractDevConsole
{
long now = System.currentTimeMillis();
recorder.sample(now, counters());
sampleChaos(now);
+ sampleBreakers(now);
+ }
+
+ /**
+ * The circuit breakers that are open or half open, from the dev consoles
of camel-resilience4j and
+ * camel-microprofile-fault-tolerance and the circuit-breaker console (the
throttling exception route policy), so
+ * this console depends on neither.
+ */
+ private void sampleBreakers(long now) {
+ DevConsoleRegistry registry =
getCamelContext().getCamelContextExtension().getContextPlugin(DevConsoleRegistry.class);
+ if (registry == null) {
+ return;
+ }
+ Map<TimelineRecorder.Breaker, Integer> states = new LinkedHashMap<>();
+ for (String[] console : BREAKER_CONSOLES) {
+ DevConsole dc = registry.resolveById(console[0]);
+ if (dc == null) {
+ continue;
+ }
+ Object json;
+ try {
+ json = dc.call(DevConsole.MediaType.JSON);
+ } catch (Exception e) {
+ LOG.debug("Failed to sample the {} dev console", console[0],
e);
+ continue;
+ }
+ if (json instanceof Map<?, ?> root && root.get("circuitBreakers")
instanceof List<?> list) {
+ for (Object o : list) {
+ if (o instanceof Map<?, ?> cb && cb.get("routeId")
instanceof String routeId) {
+ int state = breakerState(cb.get("state"));
+ if (state > 0) {
+ String id = cb.get("id") instanceof String s ? s :
null;
+ states.put(new TimelineRecorder.Breaker(routeId,
id, console[1]), state);
+ }
+ }
+ }
+ }
+ }
+ recorder.breakers(now, states);
+ }
+
+ /** 2 open (also forced open), 1 half open, else 0, whichever way the
circuit breaker spells its state. */
+ static int breakerState(Object state) {
+ // OPEN and HALF_OPEN (resilience4j, fault-tolerance), opened and half
opened (the route policy)
+ String s = state != null ?
state.toString().toUpperCase(Locale.ROOT).replaceAll("[ _-]", "") : "";
+ return switch (s) {
+ case "OPEN", "OPENED", "FORCEDOPEN" -> 2;
+ case "HALFOPEN", "HALFOPENED" -> 1;
+ default -> 0;
+ };
}
/**
@@ -272,6 +343,15 @@ public class TimelineDevConsole extends AbstractDevConsole
{
es.chaosRule() != null ? " (chaos rule " +
es.chaosRule() + ")" : ""));
}
}
+ if (r.circuitBreakers() != null) {
+ sb.append("Circuit breakers:\n");
+ for (CircuitBreakerSeries cb : r.circuitBreakers()) {
+ long open = cb.states().stream().filter(v -> v == 2).count();
+ long halfOpen = cb.states().stream().filter(v -> v ==
1).count();
+ sb.append(String.format(" %s in %s (%s): open %ds, half
open %ds%n",
+ cb.id() != null ? cb.id() : "route policy",
cb.routeId(), cb.kind(), open, halfOpen));
+ }
+ }
return sb.toString();
}
@@ -293,6 +373,8 @@ public class TimelineDevConsole extends AbstractDevConsole {
List<TimelineExchange> kept = new ArrayList<>();
List<Long> done = new ArrayList<>(count);
List<ChaosSeries> chaos = new ArrayList<>();
+ List<RouteEvent> routeEvents = new ArrayList<>();
+ List<CircuitBreakerSeries> breakers = new ArrayList<>();
// the from endpoint of each route, read before the recorder is
locked: the routes have a lock of their own, and
// the exchange events wait for the recorder
Map<String, String> fromUris = new HashMap<>();
@@ -327,6 +409,27 @@ public class TimelineDevConsole extends AbstractDevConsole
{
}
}
faults.forEach((id, v) -> chaos.add(new ChaosSeries(id,
rec.chaosRules().get(id), List.of(v))));
+ Map<TimelineRecorder.Breaker, Long[]> byBreaker = new
LinkedHashMap<>();
+ for (int i = 0; i < secs.size(); i++) {
+ TimelineRecorder.Second s = secs.get(i);
+ if (s != null && s.breakers != null) {
+ for (Map.Entry<TimelineRecorder.Breaker, Integer> e :
s.breakers.entrySet()) {
+ if (accept.test(e.getKey().routeId())) {
+ byBreaker.computeIfAbsent(e.getKey(), k ->
zeros(secs.size()))[i] = (long) e.getValue();
+ }
+ }
+ }
+ }
+ byBreaker.forEach((b, v) -> breakers.add(new
CircuitBreakerSeries(b.routeId(), b.id(), b.kind(), List.of(v))));
+ for (TimelineRecorder.Second s : secs) {
+ if (s != null && s.routeEvents != null) {
+ s.routeEvents.forEach((routeId, event) -> {
+ if (accept.test(routeId)) {
+ routeEvents.add(new RouteEvent(routeId, event,
s.second));
+ }
+ });
+ }
+ }
Map<TimelineRecorder.ErrorKind, Long[]> byKind = new
LinkedHashMap<>();
for (int i = 0; i < secs.size(); i++) {
TimelineRecorder.Second s = secs.get(i);
@@ -356,6 +459,7 @@ public class TimelineDevConsole extends AbstractDevConsole {
}
return new Response(
end, count, done, routes, errors.isEmpty() ? null : errors,
chaos.isEmpty() ? null : chaos,
+ routeEvents.isEmpty() ? null : routeEvents, breakers.isEmpty()
? null : breakers,
exchanges ? kept : null);
}
@@ -401,6 +505,17 @@ public class TimelineDevConsole extends AbstractDevConsole
{
private void onEvent(CamelEvent event) {
TimelineRecorder rec = recorder;
+ if (event instanceof CamelEvent.RouteEvent re) {
+ String what = event instanceof CamelEvent.RouteStartedEvent ?
"started"
+ : event instanceof CamelEvent.RouteStoppedEvent ? "stopped"
+ : event instanceof CamelEvent.RouteReloadedEvent ?
"reloaded"
+ : event instanceof CamelEvent.RouteRemovedEvent ? "removed"
+ : event instanceof CamelEvent.RouteRestartingFailureEvent
? "restartFailed" : null;
+ if (what != null && re.getRoute() != null) {
+ rec.routeEvent(re.getRoute().getRouteId(), what,
timestamp(event));
+ }
+ return;
+ }
if (event instanceof CamelEvent.ExchangeCreatedEvent ce) {
rec.exchangeCreated(ce.getExchange().getExchangeId(),
ce.getExchange().getFromRouteId(), timestamp(ce));
} else if (event instanceof CamelEvent.ExchangeSentEvent se) {
@@ -466,7 +581,6 @@ public class TimelineDevConsole extends AbstractDevConsole {
private TimelineEventNotifier() {
setIgnoreCamelContextInitEvents(true);
setIgnoreCamelContextEvents(true);
- setIgnoreRouteEvents(true);
setIgnoreServiceEvents(true);
setIgnoreStepEvents(true);
setIgnoreAiEvents(true);
diff --git
a/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineRecorder.java
b/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineRecorder.java
index 72288f016cc9..385feb620e75 100644
---
a/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineRecorder.java
+++
b/core/camel-console/src/main/java/org/apache/camel/impl/console/TimelineRecorder.java
@@ -70,6 +70,13 @@ final class TimelineRecorder {
List<Send> sends) {
}
+ /**
+ * A circuit breaker: of a circuit breaker EIP in a route (resilience4j or
fault-tolerance, with its id), or the
+ * throttling exception route policy of a route (route-policy, without an
id).
+ */
+ record Breaker(String routeId, String id, String kind) {
+ }
+
/** The kind of error that is counted per second. */
record ErrorKind(String routeId, String exception, Integer statusCode,
String chaosRule) {
}
@@ -122,9 +129,13 @@ final class TimelineRecorder {
final Map<ErrorKind, long[]> errors = new LinkedHashMap<>();
final List<Kept> failed = new ArrayList<>();
final List<Kept> slowest = new ArrayList<>();
+ // what happened to the routes: started, stopped, reloaded, removed,
restart failed
+ Map<String, String> routeEvents;
// with camel-chaos: the faults of each rule, and the routes a rule
had down
Map<String, Long> faults;
Set<String> down;
+ // the circuit breakers that were not closed: 2 open, 1 half open
+ Map<Breaker, Integer> breakers;
long done;
boolean closed;
@@ -271,6 +282,15 @@ final class TimelineRecorder {
}
}
+ /** A route was started, stopped, reloaded or removed: the last of a
second is kept. */
+ synchronized void routeEvent(String routeId, String event, long now) {
+ Second s = slot(now / 1000);
+ if (s.routeEvents == null) {
+ s.routeEvents = new LinkedHashMap<>();
+ }
+ s.routeEvents.put(routeId, event);
+ }
+
/** An error handler handled a failure, or an exchange failed: it is
counted as an error of its second. */
void error(ErrorKind kind, long now) {
synchronized (this) {
@@ -420,6 +440,23 @@ final class TimelineRecorder {
return chaosRules;
}
+ /**
+ * Takes the state of the circuit breakers at the given time, as the state
of the second before (the sample is taken
+ * just after a whole second): only the ones that are not closed.
+ *
+ * @param states the circuit breakers that are open (2) or half open (1)
+ */
+ synchronized void breakers(long now, Map<Breaker, Integer> states) {
+ if (states.isEmpty()) {
+ return;
+ }
+ Second s = slot(now / 1000 - 1);
+ if (s.breakers == null) {
+ s.breakers = new LinkedHashMap<>();
+ }
+ states.forEach((b, state) -> s.breakers.merge(b, state, Math::max));
+ }
+
private static long rise(long before, long now) {
return Math.max(0, now - before);
}
diff --git
a/core/camel-console/src/test/java/org/apache/camel/impl/console/TimelineDevConsoleTest.java
b/core/camel-console/src/test/java/org/apache/camel/impl/console/TimelineDevConsoleTest.java
index 19a04583e729..6552f0b8d7ed 100644
---
a/core/camel-console/src/test/java/org/apache/camel/impl/console/TimelineDevConsoleTest.java
+++
b/core/camel-console/src/test/java/org/apache/camel/impl/console/TimelineDevConsoleTest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.camel.ContextTestSupport;
import org.apache.camel.builder.RouteBuilder;
@@ -218,6 +219,88 @@ public class TimelineDevConsoleTest extends
ContextTestSupport {
assertThat(askedWithoutReport.get()).isTrue();
}
+ @Test
+ public void theSecondsACircuitBreakerWasOpen() {
+ AtomicReference<String> state = new AtomicReference<>("OPEN");
+ // camel-resilience4j is not on the classpath here: a console of its
id that answers as it does
+ AbstractDevConsole resilience = new AbstractDevConsole("camel",
"resilience4j", "Resilience", "fake") {
+ @Override
+ protected String doCallText(Map<String, Object> options) {
+ return "";
+ }
+
+ @Override
+ protected Map<String, Object> doCallJson(Map<String, Object>
options) {
+ JsonObject cb = new JsonObject();
+ cb.put("id", "cb1");
+ cb.put("routeId", "orders");
+ cb.put("state", state.get());
+ JsonObject closed = new JsonObject();
+ closed.put("id", "cb2");
+ closed.put("routeId", "batch");
+ closed.put("state", "CLOSED");
+ JsonObject root = new JsonObject();
+ root.put("circuitBreakers", List.of(cb, closed));
+ return root;
+ }
+ };
+ resilience.setCamelContext(context);
+
context.getCamelContextExtension().getContextPlugin(DevConsoleRegistry.class).register(resilience);
+ startConsole();
+
+ await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+ JsonObject out = (JsonObject)
console.call(DevConsole.MediaType.JSON, Map.of(TimelineDevConsole.SECONDS,
"10"));
+ JsonArray series = out.getCollection("circuitBreakers");
+ assertThat(series).isNotNull().hasSize(1);
+ JsonObject cb1 = (JsonObject) series.get(0);
+ assertThat(cb1.getString("id")).isEqualTo("cb1");
+ assertThat(cb1.getString("routeId")).isEqualTo("orders");
+ assertThat(cb1.getString("kind")).isEqualTo("resilience4j");
+ assertThat(cb1.<List<Number>>
getCollection("states")).contains(2L);
+ });
+ state.set("HALF_OPEN");
+ await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+ JsonObject out = (JsonObject)
console.call(DevConsole.MediaType.JSON, Map.of(TimelineDevConsole.SECONDS,
"10"));
+ JsonObject cb1 = (JsonObject) out.<JsonArray>
getCollection("circuitBreakers").get(0);
+ assertThat(cb1.<List<Number>>
getCollection("states")).contains(1L, 2L);
+ });
+ assertThat((String) console.call(DevConsole.MediaType.TEXT))
+ .contains("cb1 in orders (resilience4j): open");
+ }
+
+ @Test
+ public void aCircuitBreakerSpellsItsStateItsOwnWay() {
+ assertThat(TimelineDevConsole.breakerState("OPEN")).isEqualTo(2);
+
assertThat(TimelineDevConsole.breakerState("FORCED_OPEN")).isEqualTo(2);
+ assertThat(TimelineDevConsole.breakerState("Open")).isEqualTo(2);
+ assertThat(TimelineDevConsole.breakerState("HALF_OPEN")).isEqualTo(1);
+ assertThat(TimelineDevConsole.breakerState("HalfOpen")).isEqualTo(1);
+ assertThat(TimelineDevConsole.breakerState("opened")).isEqualTo(2);
+ assertThat(TimelineDevConsole.breakerState("half
opened")).isEqualTo(1);
+ assertThat(TimelineDevConsole.breakerState("closed")).isZero();
+ assertThat(TimelineDevConsole.breakerState(null)).isZero();
+ }
+
+ @Test
+ public void whatHappenedToTheRoutes() throws Exception {
+ startConsole();
+
+ context.getRouteController().stopRoute("failing");
+ context.getRouteController().startRoute("failing");
+
+ await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+ JsonObject out = (JsonObject)
console.call(DevConsole.MediaType.JSON);
+ JsonArray events = out.getCollection("routeEvents");
+ assertThat(events).isNotNull();
+ // the last of a second is kept: the route was started again
+ assertThat(events).anySatisfy(e -> {
+ JsonObject ev = (JsonObject) e;
+ assertThat(ev.getString("routeId")).isEqualTo("failing");
+ assertThat(ev.getString("event")).isEqualTo("started");
+ });
+ });
+ }
+
@Override
protected RouteBuilder createRouteBuilder() {
return new RouteBuilder() {
diff --git
a/docs/user-manual/modules/ROOT/images/jbang/camel-tui-timeline-circuit-breaker.png
b/docs/user-manual/modules/ROOT/images/jbang/camel-tui-timeline-circuit-breaker.png
new file mode 100644
index 000000000000..329d4ecf069f
Binary files /dev/null and
b/docs/user-manual/modules/ROOT/images/jbang/camel-tui-timeline-circuit-breaker.png
differ
diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-diagnostics.adoc
b/docs/user-manual/modules/ROOT/pages/camel-jbang-diagnostics.adoc
index b040e5420964..a8c5d7dbf83d 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-jbang-diagnostics.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-diagnostics.adoc
@@ -65,6 +65,10 @@ With xref:next@components:others:chaos.adoc[camel-chaos],
the timeline also has
second (listed by `--errors` as `chaos faults`), and the seconds a stall or
lifecycle rule had a route down (the `DOWN`
column).
+A route with a circuit breaker (the Circuit Breaker EIP with
camel-resilience4j or
+camel-microprofile-fault-tolerance, or the throttling exception route policy)
has the seconds its circuit breaker was
+open in the `CB OPEN` column: why a route went quiet, or ran its fallback,
without failing.
+
The integration records the timeline itself, for the last 5 minutes, with the
`timeline` dev console. It is on when
the dev consoles are, as with the `dev` profile that `camel run` uses by
default. The number of seconds is set with
`camel.devConsole.timeline.seconds` (60 to 3600).
diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc
b/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc
index 4692723d5c00..eeb434db5638 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc
@@ -973,7 +973,8 @@ given, and a call that names its pid is refused.
| What happened per second in the last minutes (up to 5): per route the
exchanges completed, recovered by a
redelivery, handled by the error handler and failed, with their mean, p95
and max time and the redelivery attempts;
the errors per second by route and exception; with camel-chaos, the faults
per second of each chaos rule and the
- seconds a rule had a route down; and per second the failed, handled and
recovered exchanges and the 3 slowest with the remote
+ seconds a rule had a route down; the seconds a circuit breaker was open; and
per second the failed, handled and
+ recovered exchanges and the 3 slowest with the remote
calls they made. It is recorded by the `timeline` dev console, which `camel
run` (dev profile) turns on.
| `camel_runtime_memory`
diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc
b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc
index d482709ebd13..6c52c757ead2 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc
@@ -26,18 +26,26 @@ image::jbang/camel-tui-timeline.png[The Timeline tab in an
outage of the stock s
* *Errors/s* stacks the errors per second by the kind of exception, with the
HTTP status when there is one. With
camel-chaos, the faults of each chaos rule are a layer of their own, marked ๐ฒ,
so an injected fault and a real one stay
-apart. When nothing went wrong the band is one line.
+apart. When no exchange failed, the WARN and ERROR lines of the log are the
cause instead (`log ERROR`, `log WARN`),
+read from the log file of the integration: something that went wrong outside
an exchange, such as a consumer that
+cannot connect. When nothing went wrong the band is one line.
* *Routes* has a line per route. The height is the exchanges done that second,
and the colour the worst that happened to
them: completed, recovered by redelivery, handled by the error handler (such
as the dead letter channel), or failed. A
-second a chaos rule had the route down is shaded.
+second a chaos rule had the route down is shaded, and a mark says what
happened to the route then: `โถ` started, `โ `
+stopped, `โป` reloaded, `โ` removed. A `โ` is a second a circuit breaker of the
route was open (the Circuit Breaker EIP
+with camel-resilience4j or camel-microprofile-fault-tolerance, or the
throttling exception route policy), and the route
+says `โก open 10s`: why it went quiet, or ran its fallback, without failing.
+
+image::jbang/camel-tui-timeline-circuit-breaker.png[The Timeline of the
circuit breaker example: the supplier fails three times, then the circuit
breaker of stock-check is open for 10 seconds]
* The selected route is shown in detail: its outcomes stacked per second, its
mean processing time with the p95 and max
of its slowest second, its exchanges in flight, its redelivery attempts and
its slowest remote calls. The screen opens
on the route that had the worst time.
*w* switches the window between 1, 2 and 5 minutes, and *Space* pauses so a
moment does not scroll away. Paused, *Left*
and *Right* move a cursor over the seconds (*Shift* with them goes back and
forth in the 5 minutes kept), and *Enter*
-lists the exchanges of the second at the cursor: the failed and handled ones
and the 3 slowest, with their outcome, time and
-slowest call. *Enter* again shows the selected exchange with the remote calls
it made and its exception.
+lists the exchanges of the second at the cursor: the failed, handled and
recovered ones and the 3 slowest, with their
+outcome, time and slowest call. *Enter* again shows the selected exchange with
the remote calls it made and its exception,
+or for a recovered one the failure it was redelivered for.
image::jbang/camel-tui-timeline-exchanges.png[The exchanges of a second in the
outage: the orders handled by the dead letter channel, one of them faulted by
chaos rule r1]
@@ -45,7 +53,7 @@ image::jbang/camel-tui-timeline-exchanges.png[The exchanges
of a second in the o
The integration records the timeline itself, for the last 5 minutes, with the
`timeline` dev console, which is on in
the `dev` profile of `camel run`. With camel-chaos it records the faults of
the chaos rules and the seconds they had a
-route down too. So the tab opens with the last minutes, also when the TUI
started after the
+route down too, and the seconds a circuit breaker was open. So the tab opens
with the last minutes, also when the TUI started after the
integration, and `camel get timeline` shows the same from the command line
(see xref:camel-jbang-diagnostics.adoc#_timeline[Diagnostics]).
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java
index dc491234c363..726c02446c0d 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java
@@ -336,7 +336,8 @@ public final class ToolRegistry {
+ " and max time, the exchanges in
flight and the redelivery attempts; the errors per"
+ " second by route and exception
(chaosRule names the camel-chaos rule that made"
+ " one); with camel-chaos the faults
per second of each chaos rule (chaos) and the"
- + " seconds a rule had a route down
(down of the route); and per second the failed,"
+ + " seconds a rule had a route down
(down of the route); the seconds a circuit breaker"
+ + " was open (2) or half open (1)
(circuitBreakers); and per second the failed,"
+ " handled and recovered exchanges and
the 3 slowest with the remote calls they made. Each series is"
+ " oldest first and ends at the epoch
second end. Use it to tell when something went wrong and what it did to which
route.")
.param("seconds", "integer", "The number of seconds (default
60, at most what the integration records)",
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimeline.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimeline.java
index 18c94ae9df3c..50fb94aac87b 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimeline.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimeline.java
@@ -20,8 +20,10 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import com.github.freva.asciitable.AsciiTable;
@@ -185,6 +187,16 @@ public class ListTimeline extends ProcessWatchCommand {
if (routes == null) {
return;
}
+ // the seconds a circuit breaker of the route was open
+ Map<String, Long> open = new HashMap<>();
+ JsonArray breakers = (JsonArray) timeline.get("circuitBreakers");
+ if (breakers != null) {
+ for (Object o : breakers) {
+ JsonObject cb = (JsonObject) o;
+ long seconds = Arrays.stream(values(cb, "states")).filter(v ->
v == 2).count();
+ open.merge(cb.getString("routeId"), seconds, Math::max);
+ }
+ }
for (Object o : routes) {
JsonObject r = (JsonObject) o;
RouteRow row = new RouteRow();
@@ -205,6 +217,7 @@ public class ListTimeline extends ProcessWatchCommand {
row.attempts = sum(values(r, "redeliveryAttempts"));
// with camel-chaos: the seconds a rule had the route down
row.down = r.get("down") != null ? sum(values(r, "down")) : -1;
+ row.open = breakers != null ? open.getOrDefault(row.routeId, 0L) :
-1;
row.total = row.completed + row.recovered + row.handled +
row.failed;
long[] done = new long[completed.length];
long weighted = 0;
@@ -356,6 +369,9 @@ public class ListTimeline extends ProcessWatchCommand {
jo.put("handled", r.handled);
jo.put("failed", r.failed);
jo.put("redeliveryAttempts", r.attempts);
+ if (r.open >= 0) {
+ jo.put("circuitBreakerOpenSeconds", r.open);
+ }
if (r.down >= 0) {
jo.put("downSeconds", r.down);
}
@@ -383,6 +399,9 @@ public class ListTimeline extends ProcessWatchCommand {
new Column().header("DOWN").visible(rows.stream().anyMatch(r
-> r.down >= 0))
.headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT)
.with(r -> r.down > 0 ? r.down + "s" : ""),
+ new Column().header("CB
OPEN").visible(rows.stream().anyMatch(r -> r.open > 0))
+
.headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT)
+ .with(r -> r.open > 0 ? r.open + "s" : ""),
new
Column().header("ATTEMPTS").headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT)
.with(r -> count(r.attempts)),
new
Column().header("MEAN").headerAlign(HorizontalAlign.RIGHT).dataAlign(HorizontalAlign.RIGHT)
@@ -541,6 +560,7 @@ public class ListTimeline extends ProcessWatchCommand {
long failed;
long attempts;
long down = -1;
+ long open = -1;
long mean;
long p95;
long max;
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimelineTest.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimelineTest.java
index bbfa74584b0c..d56a0a12162a 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimelineTest.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListTimelineTest.java
@@ -112,13 +112,24 @@ class ListTimelineTest extends ProcessCommandTestSupport {
assertThat(line).contains("endpoint=http:stock*", "r1", " 3 ");
}
+ @Test
+ void theSecondsACircuitBreakerOfTheRouteWasOpen() throws Exception {
+ JsonObject timeline = timeline();
+ timeline.put("circuitBreakers", Jsoner.deserialize(
+
"[{\"routeId\":\"orders\",\"id\":\"cb1\",\"kind\":\"resilience4j\",\"states\":[2,2,1]}]"));
+ String routes = run(command(timeline));
+ assertThat(routes).contains("CB OPEN");
+ assertThat(routes.lines().filter(l ->
l.contains("orders")).findFirst().orElseThrow()).contains(" 2s");
+ }
+
@Test
void withoutChaosNoDownColumn() throws Exception {
JsonObject timeline = timeline();
for (Object r : (List<?>) timeline.get("routes")) {
((JsonObject) r).remove("down");
}
- assertThat(run(command(timeline))).doesNotContain("DOWN");
+ // and no circuit breaker was open: no column for it either
+
assertThat(run(command(timeline))).doesNotContain("DOWN").doesNotContain("CB
OPEN");
}
@Test
diff --git
a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java
b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java
index 200b0d5fac33..8263d340b327 100644
---
a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java
+++
b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java
@@ -578,6 +578,7 @@ public class RuntimeTools {
p95 and max time, the exchanges in flight and the redelivery
attempts; the errors per second by \
route and exception (chaosRule names the camel-chaos rule
that made one); with camel-chaos the faults \
per second of each chaos rule (chaos) and the seconds a rule
had a route down (down of the route); \
+ the seconds a circuit breaker was open (2) or half open (1)
(circuitBreakers); \
and per second the failed, handled and recovered exchanges
(a recovered one with the failure it was \
redelivered for) and the 3 slowest with the remote calls
they made. Each series is oldest first and ends at \
the epoch second end. Use it to tell when something went
wrong and what it did to which route; \
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java
index 24b7fd940668..85740cbcac92 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java
@@ -80,6 +80,8 @@ class DataRefreshService {
// Sparkline/chart history for all metric families
private final MetricsCollector metrics = new MetricsCollector();
private final RateEstimator rates = new RateEstimator();
+ // the WARN and ERROR lines of the logs per second, for the Timeline
+ private final TimelineLog timelineLog = new TimelineLog();
// Cached PID list -- full process scan throttled to every 2 seconds (1
second in burst mode)
private volatile List<Long> cachedPids = Collections.emptyList();
@@ -325,6 +327,7 @@ class DataRefreshService {
ctx.chaosMarks.record(info);
ctx.timeline.record(info, writtenAt(pid));
mergeTimelineFile(pid);
+ timelineLog.read(info.pid, logFile(info), ctx.timeline);
}
if (info != null) {
IntegrationInfo prev = previousByPid.get(info.pid);
@@ -347,6 +350,7 @@ class DataRefreshService {
Set<String> live =
pids.stream().map(String::valueOf).collect(Collectors.toSet());
ctx.chaosMarks.retain(live);
ctx.timeline.retain(live);
+ timelineLog.retain(live);
}
if (!fullScan && ctx.selectedPid != null && !overviewTab) {
boolean checkLiveness = now - lastLivenessCheckTime >=
LIVENESS_CHECK_INTERVAL_MS;
@@ -826,6 +830,19 @@ class DataRefreshService {
}
}
+ /** The log file of the integration: by its pid, else by its name (as
Quarkus names it). */
+ private static Path logFile(IntegrationInfo info) {
+ Path dir = CommandLineHelper.getCamelDir();
+ Path byPid = dir.resolve(info.pid + ".log");
+ if (!Files.exists(byPid) && info.name != null) {
+ Path byName = dir.resolve(info.name + ".log");
+ if (Files.exists(byName)) {
+ return byName;
+ }
+ }
+ return byPid;
+ }
+
/** The last seconds of the timeline, which the integration writes each
second. */
private void mergeTimelineFile(long pid) {
try {
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java
index 11812c9ad103..c6e4892b962f 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java
@@ -63,7 +63,18 @@ final class Timeline {
* exchange completed after a redelivery in this route; the attempts are
the redeliveries tried in it.
*/
record RouteSeries(String route, long[] completed, long[] recovered,
long[] handled, long[] failed, long[] inflight,
- long[] mean, long[] p95, long[] max, long[] down, long[] attempts)
{
+ long[] mean, long[] p95, long[] max, long[] down, long[] attempts,
String[] events, long[] open) {
+
+ /** The seconds a circuit breaker of the route was open (not half
open). */
+ long openSeconds() {
+ long n = 0;
+ for (long v : open) {
+ if (v >= 2) {
+ n++;
+ }
+ }
+ return n;
+ }
long total(long[] values) {
long sum = 0;
@@ -118,9 +129,15 @@ final class Timeline {
// the exchanges kept per second, and the exchanges done per second in
the routes they came in
final TreeMap<Long, List<Kept>> exchanges = new TreeMap<>();
final TreeMap<Long, Long> done = new TreeMap<>();
+ // what happened to the routes per second: started, stopped, reloaded
+ final TreeMap<Long, Map<String, String>> routeEvents = new TreeMap<>();
// the faults per second of each chaos rule, and the routes a rule had
down
final TreeMap<Long, Map<String, Long>> faults = new TreeMap<>();
final TreeMap<Long, Set<String>> down = new TreeMap<>();
+ // the routes with a circuit breaker not closed per second: 2 open, 1
half open
+ final TreeMap<Long, Map<String, Integer>> open = new TreeMap<>();
+ // the WARN and ERROR lines of the log per second, as the TUI read
them from the log file
+ final TreeMap<Long, long[]> logs = new TreeMap<>();
final Map<String, String> ruleLabels = new LinkedHashMap<>();
// the last second the integration reported, and whether all of its
seconds were read
long end = -1;
@@ -134,8 +151,11 @@ final class Timeline {
errors.headMap(oldest).clear();
exchanges.headMap(oldest).clear();
done.headMap(oldest).clear();
+ routeEvents.headMap(oldest).clear();
faults.headMap(oldest).clear();
down.headMap(oldest).clear();
+ open.headMap(oldest).clear();
+ logs.headMap(oldest).clear();
}
}
@@ -221,6 +241,29 @@ final class Timeline {
}
rec.chaosRecorded = true;
}
+ rec.open.subMap(start, true, end, true).clear();
+ if (timeline.get("circuitBreakers") instanceof List<?> breakers) {
+ for (Object o : breakers) {
+ if (o instanceof JsonObject cb && cb.getString("routeId")
!= null) {
+ long[] states = values(cb, "states", n);
+ for (int i = 0; i < n; i++) {
+ if (states[i] > 0) {
+ rec.open.computeIfAbsent(start + i, s -> new
LinkedHashMap<>())
+ .merge(cb.getString("routeId"), (int)
states[i], Math::max);
+ }
+ }
+ }
+ }
+ }
+ rec.routeEvents.subMap(start, true, end, true).clear();
+ if (timeline.get("routeEvents") instanceof List<?> events) {
+ for (Object o : events) {
+ if (o instanceof JsonObject e && e.getString("routeId") !=
null) {
+
rec.routeEvents.computeIfAbsent(e.getLongOrDefault("second", 0), s -> new
LinkedHashMap<>())
+ .put(e.getString("routeId"),
e.getString("event"));
+ }
+ }
+ }
long[] done = values(timeline, "exchangesDone", n);
for (int i = 0; i < n; i++) {
rec.done.put(start + i, Math.max(0, done[i]));
@@ -234,8 +277,13 @@ final class Timeline {
}
}
}
+ if (full) {
+ rec.full = true;
+ } else if (rec.end >= 0 && start > rec.end + 1) {
+ // seconds were missed (the integration was not the selected
one): read all of it again
+ rec.full = false;
+ }
rec.end = Math.max(rec.end, end);
- rec.full |= full;
rec.trim(rec.end);
}
}
@@ -417,9 +465,24 @@ final class Timeline {
down[(int) (en.getKey() - start)] = 1;
}
}
+ long[] open = new long[seconds];
+ for (Map.Entry<Long, Map<String, Integer>> en : rec.open.subMap(start,
true, endSecond, true).entrySet()) {
+ Integer state = en.getValue().get(route);
+ if (state != null) {
+ open[(int) (en.getKey() - start)] = state;
+ }
+ }
+ // what happened to the route: started, stopped, reloaded, removed,
restartFailed
+ String[] events = new String[seconds];
+ for (Map.Entry<Long, Map<String, String>> en :
rec.routeEvents.subMap(start, true, endSecond, true).entrySet()) {
+ String event = en.getValue().get(route);
+ if (event != null) {
+ events[(int) (en.getKey() - start)] = event;
+ }
+ }
return new RouteSeries(
route, values[COMPLETED], values[RECOVERED], values[HANDLED],
values[FAILED],
- values[INFLIGHT], values[MEAN], values[P95], values[MAX],
down, values[ATTEMPTS]);
+ values[INFLIGHT], values[MEAN], values[P95], values[MAX],
down, values[ATTEMPTS], events, open);
}
private static long[] filled(int seconds) {
@@ -428,6 +491,16 @@ final class Timeline {
return a;
}
+ /** A WARN or ERROR line in the log of the integration, at the second of
its time. */
+ void logLine(String pid, long second, boolean error) {
+ Recording rec = byPid.computeIfAbsent(pid, k -> new Recording());
+ synchronized (rec) {
+ if (rec.end < 0 || second > rec.end - SECONDS) {
+ rec.logs.computeIfAbsent(second, s -> new long[2])[error ? 1 :
0]++;
+ }
+ }
+ }
+
/**
* The causes in the window: the errors per second by label, then the
faults of each chaos rule. Each series is
* oldest first.
@@ -453,6 +526,22 @@ final class Timeline {
errors.forEach((label, values) -> answer.add(new
CauseSeries(label, null, values)));
faults.forEach((rule, values) -> answer
.add(new CauseSeries(rec.ruleLabels.getOrDefault(rule,
rule), rule, values)));
+ if (answer.isEmpty()) {
+ // nothing failed in an exchange: what the log said went wrong
(an exception there is logged too)
+ long[] warn = new long[seconds];
+ long[] error = new long[seconds];
+ for (Map.Entry<Long, long[]> en : rec.logs.subMap(start, true,
endSecond, true).entrySet()) {
+ int i = (int) (en.getKey() - start);
+ warn[i] = en.getValue()[0];
+ error[i] = en.getValue()[1];
+ }
+ if (Arrays.stream(error).sum() > 0) {
+ answer.add(new CauseSeries("log ERROR", null, error));
+ }
+ if (Arrays.stream(warn).sum() > 0) {
+ answer.add(new CauseSeries("log WARN", null, warn));
+ }
+ }
return answer;
}
}
@@ -482,6 +571,22 @@ final class Timeline {
}
}
+ /** The last second an exchange was done in, or -1 when none in the
seconds kept. */
+ long lastActive(String pid) {
+ Recording rec = pid != null ? byPid.get(pid) : null;
+ if (rec == null) {
+ return -1;
+ }
+ synchronized (rec) {
+ for (Map.Entry<Long, Long> e :
rec.done.descendingMap().entrySet()) {
+ if (e.getValue() > 0) {
+ return e.getKey();
+ }
+ }
+ return -1;
+ }
+ }
+
/** Whether the integration reported its timeline, so an empty cause band
means no errors. */
boolean errorsSeen(String pid) {
return endSecond(pid) >= 0;
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineExchanges.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineExchanges.java
index f19100af16a8..63640fc969fc 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineExchanges.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineExchanges.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.dsl.jbang.core.commands.tui;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@@ -134,7 +136,7 @@ final class TimelineExchanges {
}
if (slowest != null) {
spans.add(Span.styled(k.calls().size() + (k.calls().size() == 1 ?
" call, " : " calls, slowest ")
- + ChaosTab.shortTime(slowest.elapsed()) + "
" + slowest.endpointUri(),
+ + ChaosTab.shortTime(slowest.elapsed()) + "
" + uri(slowest.endpointUri()),
Theme.muted()));
}
if (k.chaosRule() != null) {
@@ -165,7 +167,7 @@ final class TimelineExchanges {
if (lines.size() >= rows) {
break;
}
- lines.add(Line.from(Span.raw(" " +
pad(ChaosTab.shortTime(c.elapsed()), 8)), Span.raw(c.endpointUri())));
+ lines.add(Line.from(Span.raw(" " +
pad(ChaosTab.shortTime(c.elapsed()), 8)), Span.raw(uri(c.endpointUri()))));
}
return lines;
}
@@ -191,12 +193,24 @@ final class TimelineExchanges {
if (!spans.isEmpty()) {
spans.add(Span.styled(" ยท ", Theme.muted()));
}
- spans.add(Span.raw(e.getKey() + " "));
+ spans.add(Span.raw(uri(e.getKey()) + " "));
spans.add(Span.styled("max " +
ChaosTab.shortTime(e.getValue()[0]), Theme.info()));
});
return spans;
}
+ /** An endpoint URI as people write it: decoded (%24%7B is ${), as the
endpoint keeps it encoded. */
+ static String uri(String uri) {
+ if (uri == null || uri.indexOf('%') < 0) {
+ return uri;
+ }
+ try {
+ return URLDecoder.decode(uri.replace("+", "%2B"),
StandardCharsets.UTF_8);
+ } catch (IllegalArgumentException e) {
+ return uri;
+ }
+ }
+
private static Style style(Timeline.Kept k) {
return k.failed() ? Theme.error()
: k.handled() ? Style.EMPTY.fg(Theme.ACCENT) : k.redelivered()
? Theme.warning() : Theme.success();
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineLog.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineLog.java
new file mode 100644
index 000000000000..2783b0a0b875
--- /dev/null
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineLog.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dsl.jbang.core.commands.tui;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The WARN and ERROR lines of the log of each integration per second, for the
Timeline (CAMEL-26691): a cause when
+ * something went wrong without an exception in an exchange. Read from the log
file the TUI already shows, as it grows:
+ * the last part of it at first, then what is added at each refresh. A line
without a time of its own (a stack trace) is
+ * part of the line before and not counted.
+ */
+final class TimelineLog {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TimelineLog.class);
+
+ // how much of a log is read the first time: the lines of the last
minutes, not the whole file
+ static final int FIRST_READ = 256 * 1024;
+ // at most this much is read at a refresh, so a log that grows fast does
not hold it up
+ private static final int MAX_READ = 1024 * 1024;
+ private static final Pattern LINE = Pattern.compile(
+ "^(\\d{4}-\\d{2}-\\d{2})[
T](\\d{2}:\\d{2}:\\d{2})\\S*\\s+(WARN|ERROR)\\b");
+ private static final Pattern ANSI = Pattern.compile("\u001B\\[[;\\d]*m");
+ private static final DateTimeFormatter TIME =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ // where the log of each integration was read to
+ private final Map<String, Long> offsets = new HashMap<>();
+ private final ZoneId zone;
+
+ TimelineLog() {
+ this(ZoneId.systemDefault());
+ }
+
+ TimelineLog(ZoneId zone) {
+ this.zone = zone;
+ }
+
+ /** Reads what was added to the log of the integration since the last
time, into the timeline. */
+ void read(String pid, Path file, Timeline timeline) {
+ if (pid == null || file == null || !Files.isRegularFile(file)) {
+ return;
+ }
+ try (RandomAccessFile raf = new RandomAccessFile(file.toFile(), "r")) {
+ long size = raf.length();
+ Long known = offsets.get(pid);
+ // the first time from the last part; a log that got shorter was
rolled or cleared: from its start
+ long from = known == null ? Math.max(0, size - FIRST_READ) : known
> size ? 0 : known;
+ long to = Math.min(size, from + MAX_READ);
+ if (to <= from) {
+ offsets.put(pid, from);
+ return;
+ }
+ byte[] bytes = new byte[(int) (to - from)];
+ raf.seek(from);
+ raf.readFully(bytes);
+ // only whole lines: the rest is read the next time
+ int end = bytes.length;
+ while (end > 0 && bytes[end - 1] != '\n') {
+ end--;
+ }
+ if (end == 0 && to < size) {
+ // a line longer than the read: skip it
+ end = bytes.length;
+ }
+ String text = new String(bytes, 0, end, StandardCharsets.UTF_8);
+ // read from the middle of the file: the first line is a part of
one
+ int start = known == null && from > 0 ? text.indexOf('\n') + 1 : 0;
+ for (String line : text.substring(Math.max(0, start)).split("\n"))
{
+ count(pid, line, timeline);
+ }
+ offsets.put(pid, from + end);
+ } catch (IOException e) {
+ // the log is gone or cannot be read: nothing to count
+ LOG.debug("Cannot read the log of the integration with pid {} for
the timeline: {}", pid, e.getMessage());
+ }
+ }
+
+ private void count(String pid, String line, Timeline timeline) {
+ if (line.length() < 20) {
+ return;
+ }
+ String plain = line.indexOf('\u001B') >= 0 ?
ANSI.matcher(line).replaceAll("") : line;
+ Matcher m = LINE.matcher(plain);
+ if (m.find()) {
+ try {
+ long second = LocalDateTime.parse(m.group(1) + " " +
m.group(2), TIME).atZone(zone).toEpochSecond();
+ timeline.logLine(pid, second, "ERROR".equals(m.group(3)));
+ } catch (DateTimeParseException e) {
+ // not a time: not a line of the log format
+ }
+ }
+ }
+
+ void retain(Set<String> pids) {
+ offsets.keySet().retainAll(pids);
+ }
+}
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java
index 23d2b006045b..6875f44e8452 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java
@@ -97,6 +97,15 @@ class TimelineTab extends AbstractTableTab {
@Override
public void onTabSelected() {
refreshWhileVisible();
+ // nothing in the last minute (an app that worked at startup, a
batch): open on the window with the last exchange
+ IntegrationInfo info = ctx.findSelectedIntegration();
+ long last = info != null ? ctx.timeline.lastActive(info.pid) : -1;
+ if (pausedAt < 0 && window == 0 && last >= 0) {
+ long ago = nowSecond() - last;
+ for (int i = 0; i < WINDOWS.length && ago >= WINDOWS[window]; i++)
{
+ window = Math.min(WINDOWS.length - 1, window + 1);
+ }
+ }
// the detail opens on the route that had the worst time in the window
List<Timeline.RouteSeries> routes =
routes(ctx.findSelectedIntegration());
int worst = worst(routes);
@@ -323,7 +332,7 @@ class TimelineTab extends AbstractTableTab {
.split(area);
int inner = Math.max(20, area.width() - 2);
int label = Math.min(MAX_LABEL,
- Math.max(11, routes.stream().mapToInt(r ->
r.route().length()).max().orElse(0))) + 3;
+ Math.max(11, routes.stream().mapToInt(r ->
r.route().length()).max().orElse(0))) + 4;
// a second is 1 to 3 characters wide as the screen allows, or a
character is several seconds
int room = Math.max(10, inner - label - SUFFIX - 1);
int per = room >= seconds() ? 1 : (int) Math.ceil((double) seconds() /
room);
@@ -337,12 +346,17 @@ class TimelineTab extends AbstractTableTab {
List<Line> lines = new ArrayList<>();
renderCauses(lines, causes, anyCause, info, columns, per, label);
+ boolean anyOpen = routes.stream().anyMatch(r -> r.total(r.open()) > 0);
lines.add(Line.from(Span.raw(pad("", label)), Span.styled("Routes ",
Theme.label()),
Span.styled("โ", Theme.success()), Span.raw(" completed "),
Span.styled("โ", Theme.warning()), Span.raw(" recovered "),
Span.styled("โ", handledStyle()), Span.raw(" handled "),
Span.styled("โ", Theme.error()), Span.raw(" failed "),
- Span.styled("โ", Theme.error()), Span.raw(" down")));
+ Span.styled("โ", Theme.error()), Span.raw(" down"),
+ Span.raw(anyOpen ? " " : ""), Span.styled(anyOpen ? "โ" : "",
Theme.notice()),
+ Span.raw(anyOpen ? " circuit breaker open" : ""),
+ Span.raw(routes.stream().anyMatch(r ->
Arrays.stream(r.events()).anyMatch(e -> e != null))
+ ? " โถ started โ stopped โป reloaded" : "")));
Integer sel = tableState.selected();
routeLinesY = lines.size();
for (int i = 0; i < routes.size(); i++) {
@@ -373,6 +387,14 @@ class TimelineTab extends AbstractTableTab {
String span = seconds() >= 120 ? (seconds() / 60) + "m" : seconds() +
"s";
StringBuilder sb = new StringBuilder(" Timeline โ ").append(info.name
!= null ? info.name : info.pid)
.append(" โ last ").append(span).append(' ');
+ long last = ctx.timeline.lastActive(info.pid);
+ long quiet = nowSecond() - last;
+ if (pausedAt < 0 && last >= 0 && quiet >= seconds()) {
+ // a quiet window: when the last exchange was, out of the window
+ sb.append("โ the last exchange ").append(ChaosTab.shortTime(quiet
* 1000)).append(" ago ");
+ } else if (pausedAt < 0 && last < 0 &&
ctx.timeline.endSecond(info.pid) >= 0) {
+ sb.append("โ no exchange in the last 5m ");
+ }
if (pausedAt >= 0) {
long ago = nowSecond() - pausedAt;
sb.append("โ paused").append(ago > 0 ? ", " +
ChaosTab.shortTime(ago * 1000) + " ago " : " ");
@@ -434,15 +456,24 @@ class TimelineTab extends AbstractTableTab {
long[] handled = cols(r.handled(), false);
long[] failed = cols(r.failed(), false);
long[] down = cols(r.down(), true);
+ long[] open = cols(r.open(), true);
long[] attempts = cols(r.attempts(), false);
long max = 1;
for (int c = 0; c < columns; c++) {
max = Math.max(max, total(completed[c], recovered[c], handled[c],
failed[c]));
}
+ String[] events = eventsOf(r.events(), per);
for (int c = 0; c < columns; c++) {
long t = total(completed[c], recovered[c], handled[c], failed[c]);
- if (down[c] > 0) {
+ String event = c / width < events.length && c % width == 0 ?
events[c / width] : null;
+ if (event != null) {
+ // what happened to the route then, in place of its exchanges
+ spans.add(eventMark(event));
+ } else if (down[c] > 0) {
spans.add(Span.styled("โ", Theme.error()));
+ } else if (open[c] > 0) {
+ // the circuit breaker was open (half open dim): the calls
went to the fallback, or failed fast
+ spans.add(Span.styled("โ", open[c] >= 2 ? Theme.notice() :
Theme.muted()));
} else if (completed[c] < 0) {
spans.add(Span.raw(" "));
} else if (t == 0) {
@@ -460,6 +491,28 @@ class TimelineTab extends AbstractTableTab {
return Line.from(spans);
}
+ /** The event of each column of several seconds: the last in the column. */
+ static String[] eventsOf(String[] seconds, int per) {
+ String[] answer = new String[(seconds.length + per - 1) / per];
+ for (int i = 0; i < seconds.length; i++) {
+ if (seconds[i] != null) {
+ answer[i / per] = seconds[i];
+ }
+ }
+ return answer;
+ }
+
+ /** The mark of what happened to a route: โถ started, โ stopped, โป
reloaded, โ removed, ! restart failed. */
+ static Span eventMark(String event) {
+ return switch (event) {
+ case "started" -> Span.styled("โถ", Theme.success().bold());
+ case "stopped" -> Span.styled("โ ", Theme.warning().bold());
+ case "reloaded" -> Span.styled("โป", Theme.info().bold());
+ case "removed" -> Span.styled("โ", Theme.muted());
+ default -> Span.styled("!", Theme.error().bold());
+ };
+ }
+
/** The rate of the route over the window, and the worst that happened to
its exchanges. */
private List<Span> summary(Timeline.RouteSeries r) {
List<Span> spans = new ArrayList<>();
@@ -477,6 +530,9 @@ class TimelineTab extends AbstractTableTab {
if (r.total(r.down()) > 0) {
spans.add(Span.styled("down " + r.total(r.down()) + "s ",
Theme.error()));
}
+ if (r.openSeconds() > 0) {
+ spans.add(Span.styled("โก open " + r.openSeconds() + "s ",
Theme.notice()));
+ }
spans.add(Span.raw(rate(done, known)));
if (r.total(r.failed()) > 0) {
spans.add(Span.styled(" " + r.total(r.failed()) + " failed",
Theme.error()));
@@ -570,8 +626,10 @@ class TimelineTab extends AbstractTableTab {
long maxMean = 0;
long sumMean = 0;
int n = 0;
- for (long m : r.mean()) {
- if (m >= 0) {
+ for (int i = 0; i < r.mean().length; i++) {
+ long m = r.mean()[i];
+ // the average of the seconds that had exchanges, not of the idle
ones
+ if (m >= 0 && r.completed()[i] + r.recovered()[i] + r.handled()[i]
> 0) {
maxMean = Math.max(maxMean, m);
sumMean += m;
n++;
@@ -803,7 +861,9 @@ class TimelineTab extends AbstractTableTab {
@Override
public String description() {
- return "A timeline of the integration: errors and route outcomes per
second, with chaos rules as a layer";
+ return "A timeline of the integration: per second the errors by
exception (else the WARN and ERROR lines of the log),"
+ + " what each route did, when a route started, stopped or
reloaded, and the seconds a circuit breaker was"
+ + " open, with chaos rules as a layer; paused, the exchanges of
a second";
}
@Override
@@ -844,6 +904,7 @@ class TimelineTab extends AbstractTableTab {
row.put("handled", r.total(r.handled()));
row.put("failed", r.total(r.failed()));
row.put("downSeconds", r.total(r.down()));
+ row.put("circuitBreakerOpenSeconds", r.openSeconds());
row.put("redeliveryAttempts", r.total(r.attempts()));
JsonObject perSecond = new JsonObject();
perSecond.put("completed", array(r.completed()));
@@ -855,6 +916,23 @@ class TimelineTab extends AbstractTableTab {
perSecond.put("p95Millis", array(r.p95()));
perSecond.put("maxMillis", array(r.max()));
perSecond.put("redeliveryAttempts", array(r.attempts()));
+ // 1 a second a chaos rule had the route down; 2 a circuit breaker
of the route open, 1 half open
+ perSecond.put("down", array(r.down()));
+ perSecond.put("circuitBreaker", array(r.open()));
+ // what happened to the route: started, stopped, reloaded,
removed, restartFailed, and the second
+ JsonArray events = new JsonArray();
+ long first = endSecond() - seconds() + 1;
+ for (int i = 0; i < r.events().length; i++) {
+ if (r.events()[i] != null) {
+ JsonObject ev = new JsonObject();
+ ev.put("event", r.events()[i]);
+ ev.put("second", first + i);
+ events.add(ev);
+ }
+ }
+ if (!events.isEmpty()) {
+ row.put("events", events);
+ }
row.put("perSecond", perSecond);
rows.add(row);
}
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md
index 1ef06e712bfe..c430256bfaac 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md
@@ -9,7 +9,10 @@ went to the dead letter channel, and it recovered by itself".
The integration records the timeline itself, for the last 5 minutes (the
`timeline` dev console, on with `camel run`), so the screen opens with
the last minutes, also when the TUI was started after the integration.
-`camel get timeline` shows the same from the command line.
+When nothing happened in the last minute (an integration that did its
+work at startup), it opens on the window with the last exchange, and the
+title says how long ago that was. `camel get timeline` shows the same
+from the command line.
## Errors per second
@@ -26,6 +29,11 @@ so they are there also for an outage before the TUI started.
An
error that a rule made is not counted again as an error, so an injected
fault and a real one stay apart.
+When no exchange failed, the WARN and ERROR lines of the log are the
+cause instead (`log ERROR`, `log WARN`): something that went wrong outside
+an exchange, such as a consumer that cannot connect. They are read from
+the log file, from the last part of it when the TUI starts.
+
When nothing went wrong, the band is one line: `Errors/s none`.
## Routes
@@ -42,7 +50,14 @@ The colour is the worst that happened to them that second:
- red: failed
A `ยท` is a second with no exchange, and a red `โ` is a second a chaos
-rule had the route down (`stall` or a lifecycle fault). At the right: the
+rule had the route down (`stall` or a lifecycle fault). A `โ` is a second
+a circuit breaker of the route was open (dim: half open), so its calls
+went to the fallback or failed fast, and the route says `โก open 12s`:
+the Circuit Breaker EIP (camel-resilience4j,
+camel-microprofile-fault-tolerance) or the throttling exception route
+policy. A mark says what
+happened to the route then: `โถ` started, `โ ` stopped, `โป` reloaded, `โ`
+removed, `!` a restart failed. At the right: the
exchanges per second over the window, and the worst count.
## The selected route
@@ -57,11 +72,12 @@ slowest exchanges. The last line has the totals of the
window.
Paused, a cursor `โฒ` under the time axis picks a second (or a column of
seconds). **Enter** lists the exchanges the integration kept of it: the
-failed and handled ones (up to 10 a second) and the 3 slowest, with their
outcome,
-time and slowest call, and the chaos rule that faulted one. **Enter**
-again shows the selected exchange: its calls with their times and its
-exception. The title says how many exchanges were done then, of which
-these were kept.
+failed, handled and recovered ones (up to 10 a second) and the 3 slowest,
+with their outcome, time and slowest call, and the chaos rule that
+faulted one. **Enter** again shows the selected exchange: its calls with
+their times and its exception, or for a recovered one the failure it was
+redelivered for (`recovered after:`). The title says how many exchanges
+were done then, of which these were kept.
## Keys
@@ -80,5 +96,6 @@ these were kept.
- **Shift+F8** โ Explain with AI what happened, as cause and effect
An AI can read the same per second series with `tui_get_table` on the
-Timeline tab, and the exchanges of the second at the cursor while the
-list is open.
+Timeline tab (with the causes, the seconds a route was down or its
+circuit breaker open, and when a route started, stopped or reloaded),
+and the exchanges of the second at the cursor while the list is open.
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineJson.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineJson.java
index e0ec2db316a8..d039b5491595 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineJson.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineJson.java
@@ -34,6 +34,7 @@ final class TimelineJson {
private final Map<String, Map<String, long[]>> routes = new
LinkedHashMap<>();
private final List<JsonObject> errors = new ArrayList<>();
private final List<JsonObject> exchanges = new ArrayList<>();
+ private final List<JsonObject> breakers = new ArrayList<>();
private final long[] done;
TimelineJson(long end, int seconds) {
@@ -66,6 +67,17 @@ final class TimelineJson {
return this;
}
+ /** A circuit breaker of a route, per second 2 open, 1 half open, 0
closed. */
+ TimelineJson breaker(String routeId, String id, long[] states) {
+ JsonObject cb = new JsonObject();
+ cb.put("routeId", routeId);
+ cb.put("id", id);
+ cb.put("kind", "resilience4j");
+ cb.put("states", list(states));
+ breakers.add(cb);
+ return this;
+ }
+
/** An exchange kept, done at the second i. */
TimelineJson exchange(
String id, String routeId, int i, long elapsed, String outcome,
String exception, String chaosRule,
@@ -117,6 +129,9 @@ final class TimelineJson {
if (!errors.isEmpty()) {
root.put("errors", errors);
}
+ if (!breakers.isEmpty()) {
+ root.put("circuitBreakers", breakers);
+ }
root.put("exchanges", exchanges);
try {
return (JsonObject) Jsoner.deserialize(root.toJson());
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineLogTest.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineLogTest.java
new file mode 100644
index 000000000000..6479d4e8951c
--- /dev/null
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineLogTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dsl.jbang.core.commands.tui;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class TimelineLogTest {
+
+ // 2027-01-15 08:00:00 UTC
+ private static final long T0 = LocalDateTime.of(2027, 1, 15, 8, 0,
0).toEpochSecond(ZoneOffset.UTC);
+
+ @TempDir
+ Path dir;
+
+ private final Timeline timeline = new Timeline();
+ private final TimelineLog log = new TimelineLog(ZoneOffset.UTC);
+
+ private static String line(int second, String level, String message) {
+ return String.format("2027-01-15 08:00:%02d.123 %-5s 4711 --- [main]
org.acme.Shop : %s%n", second, level, message);
+ }
+
+ private long[] series(String label) {
+ return timeline.causes("4711", T0 + 59, 60).stream().filter(c ->
c.label().equals(label))
+ .map(Timeline.CauseSeries::values).findFirst().orElse(new
long[60]);
+ }
+
+ @Test
+ void theWarnAndErrorLinesPerSecondAsTheLogGrows() throws Exception {
+ Path file = dir.resolve("4711.log");
+ Files.writeString(file, line(1, "INFO", "started") + line(2, "WARN",
"slow") + line(2, "WARN", "slower")
+ + line(3, "ERROR", "boom")
+ + "java.lang.IllegalStateException: boom\n\tat
org.acme.Shop.run(Shop.java:1)\n");
+ log.read("4711", file, timeline);
+ assertThat(series("log WARN")[2]).isEqualTo(2);
+ assertThat(series("log ERROR")[3]).isEqualTo(1);
+ // the stack trace is part of its line
+ assertThat(Arrays.stream(series("log ERROR")).sum()).isEqualTo(1);
+
+ // what is added is read the next time, a line that is not whole yet
the time after
+ Files.writeString(file, line(5, "ERROR", "again") + "2027-01-15
08:00:06.000 WARN 4711 --- [main] half",
+ StandardOpenOption.APPEND);
+ log.read("4711", file, timeline);
+ assertThat(series("log ERROR")[5]).isEqualTo(1);
+ assertThat(series("log WARN")[6]).isZero();
+ Files.writeString(file, " a line\n", StandardOpenOption.APPEND);
+ log.read("4711", file, timeline);
+ assertThat(series("log WARN")[6]).isEqualTo(1);
+ // read once only
+ log.read("4711", file, timeline);
+ assertThat(Arrays.stream(series("log WARN")).sum()).isEqualTo(3);
+ }
+
+ @Test
+ void aBigLogIsReadFromItsLastPart() throws Exception {
+ Path file = dir.resolve("4711.log");
+ StringBuilder sb = new StringBuilder();
+ String old = line(10, "ERROR", "long ago");
+ while (sb.length() < TimelineLog.FIRST_READ * 2) {
+ sb.append(old);
+ }
+ sb.append(line(20, "WARN", "now"));
+ Files.writeString(file, sb.toString());
+ log.read("4711", file, timeline);
+ assertThat(series("log WARN")[20]).isEqualTo(1);
+ // only the lines of the last part, and not the line cut in two at its
start
+ long errors = Arrays.stream(series("log ERROR")).sum();
+ assertThat(errors).isLessThan((long) (TimelineLog.FIRST_READ /
old.length()) + 1).isGreaterThan(0);
+ }
+
+ @Test
+ void theLogIsNotACauseWhenAnExchangeFailed() throws Exception {
+ TimelineJson t = new TimelineJson(T0 + 59, 60);
+ long[] counts = new long[60];
+ counts[3] = 1;
+ t.route("orders", "failed", 3, 1).error("orders",
"java.io.IOException", null, null, counts);
+ timeline.merge("4711", t.json(), true);
+ Path file = dir.resolve("4711.log");
+ Files.write(file, List.of(line(3, "ERROR", "failed").trim()),
StandardCharsets.UTF_8);
+ log.read("4711", file, timeline);
+ assertThat(timeline.causes("4711", T0 + 59,
60)).extracting(Timeline.CauseSeries::label)
+ .containsExactly("IOException");
+ }
+
+ @Test
+ void colouredLinesCountToo() throws Exception {
+ Path file = dir.resolve("4711.log");
+ Files.writeString(file, "\u001B[2m2027-01-15 08:00:04.000\u001B[0m
\u001B[33m WARN\u001B[0m 4711 --- [main] x\n");
+ log.read("4711", file, timeline);
+ assertThat(series("log WARN")[4]).isEqualTo(1);
+ }
+}
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java
index 23786f7d5f07..257389c061fd 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java
@@ -172,6 +172,36 @@ class TimelineTabRenderTest {
assertThat(screen).contains("Errors/s none").contains("idle");
}
+ @Test
+ void aQuietMinuteOpensOnTheWindowWithTheLastExchange() {
+ IntegrationInfo batch = new IntegrationInfo();
+ batch.pid = "88";
+ batch.name = "batch";
+ batch.routes.add(route("files", 0, 0));
+ MonitorContext bctx = new MonitorContext(new
AtomicReference<>(List.of(batch)), new AtomicReference<>(List.of()));
+ bctx.selectedPid = "88";
+ // the files were picked up at startup, 3 minutes ago
+ bctx.timeline.merge("88", new TimelineJson(T0 / 1000 + 59,
300).route("files", "completed", 60, 3)
+ .route("files", "meanMillis", 60, 40).route("files",
"maxMillis", 60, 90).json(), true);
+ TimelineTab tab = new TimelineTab(bctx, now::get);
+
+ tab.onTabSelected();
+
+ assertThat(tab.seconds()).isEqualTo(300);
+ String screen = TuiTestHelper.renderToString(tab, 160, 40);
+ assertThat(screen).contains("last 5m").contains("avg 40ms");
+ tab.handleKeyEvent(KeyEvent.ofChar('w'));
+ assertThat(TuiTestHelper.renderToString(tab, 160, 40)).contains("the
last exchange 3m59s ago");
+ }
+
+ @Test
+ void theCallsAreShownDecoded() {
+
assertThat(TimelineExchanges.uri("file://parked?fileName=%24%7Bheader.CamelFileName%7D"))
+ .isEqualTo("file://parked?fileName=${header.CamelFileName}");
+
assertThat(TimelineExchanges.uri("http://stock?q=a+b";)).isEqualTo("http://stock?q=a+b";);
+
assertThat(TimelineExchanges.uri("http://stock";)).isEqualTo("http://stock";);
+ }
+
@Test
void theScreenOpensOnTheRouteWithTheWorstTime() {
TimelineTab tab = new TimelineTab(ctx, now::get);
@@ -227,6 +257,30 @@ class TimelineTabRenderTest {
assertThat(TuiTestHelper.renderToString(tab, 160,
40)).contains("recovered after: Connection refused");
}
+ @Test
+ void theSecondsACircuitBreakerWasOpen() {
+ long[] states = new long[60];
+ for (int i = 40; i < 52; i++) {
+ states[i] = i < 50 ? 2 : 1;
+ }
+ TimelineJson t = TimelineTabRenderTest.outage(T0 / 1000 + 59, 60,
0).breaker("orders", "cb1", states);
+ ctx.timeline.merge("1234", t.json(), true);
+ TimelineTab tab = new TimelineTab(ctx, now::get);
+ // the bolt is two columns wide: the screen has a blank cell after it
+ String screen = TuiTestHelper.renderToString(tab, 160, 40).replace("โก
", "โก");
+ assertThat(screen).contains("โ circuit breaker open").contains("โก open
10s");
+ String orders = screen.lines().filter(l -> l.contains("orders") &&
l.contains("โก")).findFirst().orElseThrow();
+ assertThat(orders.chars().filter(c -> c ==
'โ').count()).isGreaterThanOrEqualTo(12);
+ assertThat(screen.lines().filter(l ->
l.contains("shipping")).findFirst().orElseThrow()).doesNotContain("โ");
+
+ // the same for an agent (tui_get_table)
+ JsonObject row = ((JsonArray)
tab.getTableDataAsJson().get("rows")).stream().map(JsonObject.class::cast)
+ .filter(r ->
"orders".equals(r.getString("route"))).findFirst().orElseThrow();
+ assertThat(row.getLong("circuitBreakerOpenSeconds")).isEqualTo(10);
+ JsonArray recorded = (JsonArray) ((JsonObject)
row.get("perSecond")).get("circuitBreaker");
+ assertThat(recorded.stream().filter(v -> ((Number) v).longValue() ==
2).count()).isEqualTo(10);
+ }
+
@Test
void enterShowsTheExchangesOfTheSecondAtTheCursorAndOneInDetail() {
TimelineTab tab = new TimelineTab(ctx, now::get);
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java
index 56bf811ff37a..c8880cc97e2a 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java
@@ -156,6 +156,34 @@ class TimelineTest {
assertThat(Timeline.ruleLabel("r2", "route=shipping stall
duration=15s")).isEqualTo("r2 stall (shipping)");
}
+ @Test
+ void missedSecondsAreReadAgain() {
+ timeline.merge("1234", new TimelineJson(S0 + 9, 10).route("orders",
"completed", 9, 1).json(), true);
+ assertThat(timeline.isFull("1234")).isTrue();
+ // the next seconds come right after: nothing missed
+ timeline.merge("1234", new TimelineJson(S0 + 12, 3).route("orders",
"completed", 2, 1).json(), false);
+ assertThat(timeline.isFull("1234")).isTrue();
+ // the integration was not the selected one for a while: seconds missed
+ timeline.merge("1234", new TimelineJson(S0 + 40, 3).route("orders",
"completed", 2, 1).json(), false);
+ assertThat(timeline.isFull("1234")).isFalse();
+ assertThat(timeline.lastActive("1234")).isEqualTo(S0 + 40);
+ }
+
+ @Test
+ void whatHappenedToTheRoutes() throws Exception {
+ JsonObject json = new TimelineJson(S0 + 2, 3).route("orders",
"completed", 0, 1).json();
+ JsonObject stopped = new JsonObject();
+ stopped.put("routeId", "orders");
+ stopped.put("event", "stopped");
+ stopped.put("second", S0 + 1);
+ json.put("routeEvents", List.of(stopped));
+ timeline.merge("1234", json, true);
+
+ assertThat(orders(S0 + 2, 3).events()).containsExactly(null,
"stopped", null);
+ assertThat(TimelineTab.eventsOf(new String[] { null, "stopped",
"started", null }, 2))
+ .containsExactly("stopped", "started");
+ }
+
@Test
void notATimelineIsIgnored() {
timeline.merge("1234", new JsonObject(), true);
