This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26788-chaos-explore-save in repository https://gitbox.apache.org/repos/asf/camel.git
commit bf46311ac03c48a9520b2759576719814af596d9 Author: Claus Ibsen <[email protected]> AuthorDate: Mon Nov 9 17:34:10 2026 +0100 CAMEL-26788: camel-jbang - camel cmd chaos-explore saves its report, and a replayable scenario per failed experiment, in test/chaos Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../camel-jbang-cmd-chaos-explore.adoc | 1 + .../META-INF/camel-jbang-commands-metadata.json | 2 +- .../apache/camel/dsl/jbang/core/commands/Ask.java | 6 + .../dsl/jbang/core/commands/ChaosExplore.java | 165 ++++++++++++++++++++- .../core/commands/ai/ChaosExplorePlaybook.java | 15 +- .../dsl/jbang/core/commands/ChaosExploreTest.java | 77 +++++++++- .../src/test/resources/golden/cli-help.txt | 11 +- 7 files changed, 262 insertions(+), 15 deletions(-) diff --git a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-explore.adoc b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-explore.adoc index 8ddbf5b6a012..eb684864ce1a 100644 --- a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-explore.adoc +++ b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-explore.adoc @@ -25,6 +25,7 @@ camel cmd chaos-explore [options] | `--focus` | What to explore, such as a route id (default: all routes) | | String | `--max-iterations` | Maximum number of tool-calling rounds | 100 | int | `--model` | Model to use. Default: the provider's default; with Ollama the installed model best at tool calling | | String +| `--save` | Save the report in test/chaos/reports (as markdown, and the journal as JSON), and each experiment that failed as a scenario in test/chaos, to replay it | true | boolean | `--show-tools` | Show the tool calls and results as they happen | | boolean | `--timeout` | Timeout in seconds for one answer of the LLM (an experiment can take minutes) | 300 | int | `--tools` | The tools the model gets: core (a small set, for a local model), full, or auto | auto | ToolSet diff --git a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json index b7bba9ca7fc9..a98548c7bc7f 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json +++ b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json @@ -3,7 +3,7 @@ { "name": "ask", "fullName": "ask", "description": "Ask a question about a running Camel application using AI", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Ask", "options": [ { "names": "--api-key", "description": "API key. Also reads ANTHROPIC_API_KEY, OPENAI_API_KEY, WATSONX_APIKEY, or LLM_API_KEY env vars", "javaType": "java.lang.String", "type": "string" }, { "names": "--api-type", "description": "API type: 'ollama', 'openai', 'anthropic', or 'watsonx'", "javaType": [...] { "name": "bind", "fullName": "bind", "description": "DEPRECATED: Bind source and sink Kamelets as a new Camel integration", "deprecated": true, "sourceClass": "org.apache.camel.dsl.jbang.core.commands.bind.Bind", "options": [ { "names": "--error-handler", "description": "Add error handler (none|log|sink:<endpoint>). Sink endpoints are expected in the format \"[[apigroup\/]version:]kind:[namespace\/]name\", plain Camel URIs or Kamelet name.", "javaType": "java.lang.String", "type": " [...] { "name": "catalog", "fullName": "catalog", "description": "List artifacts from Camel Catalog", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.catalog.CatalogCommand", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "component", "fullName": "catalog component", "description": "List components from the Camel Catalog", "sourceClass": "org.apache.camel.dsl.jbang.co [...] - { "name": "cmd", "fullName": "cmd", "description": "Performs commands in the running Camel integrations, such as start\/stop route, or change logging levels.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelAction", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "browse", "fullName": "cmd browse", "description": "Browse pending messages on endpoints [...] + { "name": "cmd", "fullName": "cmd", "description": "Performs commands in the running Camel integrations, such as start\/stop route, or change logging levels.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelAction", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "browse", "fullName": "cmd browse", "description": "Browse pending messages on endpoints [...] { "name": "completion", "fullName": "completion", "description": "Generate completion script for bash\/zsh", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Complete", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ] }, { "name": "config", "fullName": "config", "description": "Get and set user configuration values", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.config.ConfigCommand", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "get", "fullName": "config get", "description": "Display user configuration value", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.config. [...] { "name": "debug", "fullName": "debug", "description": "Debug local Camel integration", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Debug", "options": [ { "names": "--ago", "description": "Use ago instead of yyyy-MM-dd HH:mm:ss in timestamp.", "javaType": "boolean", "type": "boolean" }, { "names": "--background", "description": "Run in the background", "defaultValue": "false", "javaType": "boolean", "type": "boolean" }, { "names": "--background-wait", "description": "How [...] diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java index dbdccd7ca25b..fef688a641cd 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java @@ -131,6 +131,10 @@ public class Ask extends CamelCommand { // the tools the model does not get, such as starting a chaos exploration that camel cmd chaos-explore started Set<String> hiddenTools = Set.of(); + // the last answer of the model, such as the report of camel cmd chaos-explore, which it saves + String lastAnswer; + // the model that answered, and its API, such as qwen3.6 (ollama) + String usedModel; private long targetPid; private AskTools askTools; @@ -170,6 +174,7 @@ public class Ask extends CamelCommand { return 1; } warnSmallModel(client.model, client.ollamaParameterBillions()); + usedModel = client.model + " (" + client.apiType + ")"; RuntimeHelper.ProcessInfo process = findProcess(nameOrPid); if (process != null) { @@ -296,6 +301,7 @@ public class Ask extends CamelCommand { continue; } messages.add(LlmClient.Message.assistantWithToolCalls(response.text(), List.of())); + lastAnswer = response.text(); printStats(totalUsage, startTime); return 0; } diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ChaosExplore.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ChaosExplore.java index 5800dded6098..dabb039c7024 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ChaosExplore.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ChaosExplore.java @@ -16,6 +16,11 @@ */ package org.apache.camel.dsl.jbang.core.commands; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -31,6 +36,7 @@ import org.apache.camel.dsl.jbang.core.common.ProcessFinder.CamelProcess; import org.apache.camel.dsl.jbang.core.common.RuntimeHelper; import org.apache.camel.util.TimeUtils; import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; import picocli.CommandLine.Command; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -110,6 +116,11 @@ public class ChaosExplore extends CamelCommand { description = "Show the tool calls and results as they happen") boolean showTools; + @Option(names = { "--save" }, negatable = true, defaultValue = "true", fallbackValue = "true", + description = "Save the report in test/chaos/reports (as markdown, and the journal as JSON), and each" + + " experiment that failed as a scenario in test/chaos, to replay it") + boolean save = true; + @Option(names = { "--verbose" }, description = "Print debug information of the LLM calls") boolean verbose; @@ -133,30 +144,52 @@ public class ChaosExplore extends CamelCommand { printer().printErr(started.error()); return started.exitCode(); } - printer().println("Let there be chaos: " + process.name() + " (pid " + process.pid() + "), " + String integration = displayName(process); + printer().println("Let there be chaos: " + integration + " (pid " + process.pid() + "), " + TimeUtils.printDuration(started.reply().getLongOrDefault("budget", 0)) + (focus != null && !focus.isBlank() ? ", focus: " + focus : "")); int answer; + Ask ask = ask(process, integration); try { - answer = ask(process).doCall(); + answer = ask.doCall(); } finally { // the AI may have finished it already; else its rules go now ActionClient.execute(process, "chaos", root -> root.put("command", "explore-finish"), 10000); } - JsonObject exploration = ChaosExperimentRun.exploration(RuntimeHelper.readStatus(process.pid())); + JsonObject status = RuntimeHelper.readStatus(process.pid()); + JsonObject exploration = ChaosExperimentRun.exploration(status); List<String> lines = journal(exploration); printer().println(); lines.forEach(printer()::println); + if (save && exploration != null) { + Saved saved = save(Path.of("test", "chaos"), LocalDateTime.now(), + new Run(integration, budget, focus, ask.usedModel, seed(status)), exploration, ask.lastAnswer, + lines); + printer().println(); + printer().println("Report: " + saved.report()); + saved.scenarios().forEach(s -> printer().println("Scenario: " + s + " (replay: camel run * --chaos=" + s + ")")); + } if (failed(exploration)) { return CliExitCode.FAILURE; } return answer; } + /** The name of the integration as camel get chaos has it: its CamelContext, else the name of the process. */ + static String displayName(CamelProcess process) { + if (process.status() != null && process.status().get("context") instanceof JsonObject context) { + String name = context.getString("name"); + if (name != null && !name.isBlank() && !"CamelJBang".equals(name)) { + return name; + } + } + return process.name(); + } + /** camel ask, with the playbook as its question, for the integration. */ - Ask ask(CamelProcess process) { + Ask ask(CamelProcess process, String integration) { Ask ask = new Ask(getMain()); - ask.question = List.of(ChaosExplorePlaybook.text(ChaosExplorePlaybook.ASK, process.name(), budget, focus, true)); + ask.question = List.of(ChaosExplorePlaybook.text(ChaosExplorePlaybook.ASK, integration, budget, focus, true)); ask.nameOrPid = Long.toString(process.pid()); ask.url = url; ask.apiType = apiType; @@ -174,6 +207,128 @@ public class ChaosExplore extends CamelCommand { return ask; } + /** What the exploration ran with, for the report. */ + record Run(String integration, String budget, String focus, String model, Long seed) { + } + + /** The files written: the report (markdown, with the journal as JSON next to it), and a scenario per failure. */ + record Saved(Path report, Path journal, List<Path> scenarios) { + } + + private static Long seed(JsonObject status) { + if (status != null && status.get("chaos") instanceof JsonObject chaos + && chaos.get("report") instanceof JsonObject report && report.get("seed") instanceof Number n) { + return n.longValue(); + } + return null; + } + + /** + * Saves what the exploration found (CAMEL-26788): the report of the AI with the journal of the experiments as + * markdown in reports/, the journal as JSON next to it, and each experiment that failed as a scenario, so camel run + * --chaos or camel cmd chaos-load replays its verdict. + * + * @param dir the chaos folder of the project, test/chaos, which camel run and export leave alone + */ + static Saved save( + Path dir, LocalDateTime when, Run run, JsonObject exploration, String report, List<String> journal) + throws IOException { + String stamp = when.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")); + String date = when.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")); + Path reports = Files.createDirectories(dir.resolve("reports")); + List<Path> scenarios = new ArrayList<>(); + List<Object> experiments = exploration.getCollection("experiments"); + if (experiments != null) { + for (Object o : experiments) { + JsonObject e = (JsonObject) o; + if (!"FAIL".equals(e.getString("verdict"))) { + continue; + } + Path scenario = dir.resolve("explore-" + stamp + "-" + e.getIntegerOrDefault("number", 0) + ".chaos"); + Files.writeString(scenario, scenario(e, run, date, scenario)); + scenarios.add(scenario); + } + } + + StringBuilder md = new StringBuilder(); + md.append("# Chaos exploration: ").append(run.integration()).append("\n\n"); + md.append(date).append(" · budget ").append(run.budget()); + if (run.focus() != null && !run.focus().isBlank()) { + md.append(" · focus ").append(run.focus()); + } + if (run.model() != null) { + md.append(" · ").append(run.model()); + } + if (run.seed() != null) { + md.append(" · seed ").append(run.seed()); + } + md.append("\n\n## Report\n\n").append(report != null && !report.isBlank() + ? report.strip() : "(the AI wrote no report)").append("\n\n## Experiments\n\n```text\n"); + journal.forEach(line -> md.append(line).append('\n')); + md.append("```\n"); + if (!scenarios.isEmpty()) { + md.append("\n## Scenarios\n\nEach experiment that failed, to replay its verdict:\n\n"); + for (Path s : scenarios) { + md.append("- `").append(s).append("`: `camel run * --chaos=").append(s).append("`\n"); + } + } + Path markdown = reports.resolve("explore-" + stamp + ".md"); + Files.writeString(markdown, md.toString()); + + JsonObject json = new JsonObject(); + json.put("integration", run.integration()); + json.put("budget", run.budget()); + if (run.focus() != null) { + json.put("focus", run.focus()); + } + if (run.model() != null) { + json.put("model", run.model()); + } + if (run.seed() != null) { + json.put("seed", run.seed()); + } + json.put("report", report); + json.put("exploration", exploration); + json.put("scenarios", scenarios.stream().map(Path::toString).toList()); + Path journalFile = reports.resolve("explore-" + stamp + ".json"); + Files.writeString(journalFile, Jsoner.prettyPrint(json.toJson())); + return new Saved(markdown, journalFile, scenarios); + } + + /** A failed experiment as a scenario: what it tested and found as comments, the seed, and its rules. */ + static String scenario(JsonObject experiment, Run run, String date, Path file) { + StringBuilder sb = new StringBuilder(); + sb.append("# From the chaos exploration of ").append(run.integration()).append(" on ").append(date) + .append(", experiment ").append(experiment.getIntegerOrDefault("number", 0)).append(": ") + .append(experiment.getString("verdict")).append('\n'); + String hypothesis = experiment.getString("hypothesis"); + if (hypothesis != null) { + sb.append("# hypothesis: ").append(hypothesis).append('\n'); + } + List<Object> expectations = experiment.getCollection("expectations"); + if (expectations != null) { + for (Object o : expectations) { + JsonObject x = (JsonObject) o; + sb.append("# expect ").append(x.getString("expect")).append(": ").append(x.getString("state")) + .append(", ").append(x.getString("value")).append('\n'); + } + } + String note = experiment.getString("note"); + if (note != null) { + sb.append("# note: ").append(note).append('\n'); + } + sb.append("# replay: camel run * --chaos=").append(file) + .append(" (and send it messages as the experiment did, when its routes do not feed themselves)\n"); + if (run.seed() != null) { + sb.append("seed=").append(run.seed()).append('\n'); + } + List<Object> rules = experiment.getCollection("ruleTexts"); + if (rules != null) { + rules.forEach(r -> sb.append(r).append('\n')); + } + return sb.toString(); + } + /** The journal of the exploration: how it ended, and a line per experiment with its expectations. */ static List<String> journal(JsonObject exploration) { List<String> lines = new ArrayList<>(); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExplorePlaybook.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExplorePlaybook.java index 33a80209ac75..dc0e24ba7f08 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExplorePlaybook.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExplorePlaybook.java @@ -81,19 +81,24 @@ public final class ChaosExplorePlaybook { - hypothesis: what should hold, in a few words, such as "a failed refund is retried, not lost" - rule: the suggested rule with a short until= (such as until=30s), and one expectation that \ decides the hypothesis, with within= a little longer than until=: expect=<route>:failed<=0 \ - (nothing fails), expect=<route>-><next route>:lost<=0 (nothing lost between two routes), \ + (nothing fails), expect=<route>-><next route>:lost<=0 (nothing lost when a route hands messages to \ + the next route, as the suggestions of kind lost name them), \ expect=<route>:20 (20 completed), expect=<route>:maxTime<=5s (none slower than 5s) - - feed: when the feed of the suggestion is kind send, its endpoint, with body, repeat (such as \ - 20) and interval (such as 100..300); leave feed out when the route feeds itself (kind self). \ + - feed: a route that does not feed itself gets no messages unless you give feed. When the feed \ + of the suggestion is kind send, always give its endpoint, with body (a message as the route \ + expects it, such as JSON), repeat (such as 20) and interval (such as 100..300); leave feed out \ + only when the route feeds itself (kind self). \ Routes fed over http or from files cannot be fed by an experiment yet: try them only when they \ feed themselves + - a rule at=dlq fails the dead letter channel, so it fires only when a message goes there: use \ + the combined suggestions, which add the fault that sends messages there too - wait: longer than the within= of the expectation - note: what you make of the result, in one sentence 4. Read the result: the verdict (FAIL, PASS or INCONCLUSIVE), the expectations, what the routes \ did, and the traffic. On FAIL, call %s for the failed exchanges and their exceptions, and say why it \ broke. On INCONCLUSIVE, run it again with a longer wait. On PASS, move on. - 5. Go on while timeLeft of the result is more than the next experiment needs (its until= and wait, \ - plus a minute). Stop with about a minute left. + 5. Go on while timeLeft of the result (millis) is more than the next experiment needs: its wait \ + plus a minute. When it is less, do not begin another experiment: finish. 6. Finish with %s, and write the report: - Summary: how many experiments, how many held and how many broke - Findings: per FAIL the route, what broke, the evidence (the expectation and its actual value, \ diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ChaosExploreTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ChaosExploreTest.java index fa94b0c72251..902a97c7d675 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ChaosExploreTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ChaosExploreTest.java @@ -16,13 +16,18 @@ */ package org.apache.camel.dsl.jbang.core.commands; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; import java.util.List; import java.util.Map; import org.apache.camel.dsl.jbang.core.common.ProcessFinder.CamelProcess; import org.apache.camel.util.json.JsonArray; import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import picocli.CommandLine; import static org.assertj.core.api.Assertions.assertThat; @@ -39,7 +44,7 @@ class ChaosExploreTest { CommandLine.populateCommand(command, "shop", "--budget=30m", "--focus=refunds", "--model=qwen3.6", "--api-type=ollama", "--tools=core", "--show-tools"); - Ask ask = command.ask(new CamelProcess(4711, "shop", null)); + Ask ask = command.ask(new CamelProcess(4711, "shop.camel.yaml", null), "shop"); assertThat(ask.question).singleElement().asString() .contains("explore how the integration shop copes", "The exploration is running (30m)", "chaos_experiment"); @@ -56,6 +61,17 @@ class ChaosExploreTest { assertThat(ask.hiddenTools).containsExactly("chaos_explore_start"); } + @Test + void theIntegrationIsNamedAsCamelGetChaosNamesIt() { + // camel run * names the process after its files, and the CamelContext by --name + JsonObject status = new JsonObject(Map.of("context", new JsonObject(Map.of("name", "shop")))); + assertThat(ChaosExplore.displayName(new CamelProcess(1, "chaos.camel.yaml parked.camel.yaml", status))) + .isEqualTo("shop"); + JsonObject unnamed = new JsonObject(Map.of("context", new JsonObject(Map.of("name", "CamelJBang")))); + assertThat(ChaosExplore.displayName(new CamelProcess(1, "shop.camel.yaml", unnamed))).isEqualTo("shop.camel.yaml"); + assertThat(ChaosExplore.displayName(new CamelProcess(1, "shop.camel.yaml", null))).isEqualTo("shop.camel.yaml"); + } + @Test void theJournalWhenTheAiIsDone() { JsonObject failed = new JsonObject( @@ -80,4 +96,63 @@ class ChaosExploreTest { assertThat(ChaosExplore.journal(null)).containsExactly( "No journal of the exploration in the status of the integration"); } + + @Test + void whatTheExplorationFoundIsSaved(@TempDir Path dir) throws Exception { + JsonObject failed = new JsonObject( + Map.of("number", 1, "verdict", "FAIL", "open", false, "endedBy", "ended", + "hypothesis", "nothing is lost after the hand-off", "rules", new JsonArray(List.of("r1", "r2")), + "ruleTexts", new JsonArray( + List.of("route=shipping error rate=0.2 until=30s" + + " expect=orders->shipping:lost<=0 within=45s", + "endpoint=seda:dlq error at=dlq until=30s")), + "expectations", new JsonArray( + List.of(new JsonObject( + Map.of("expect", "orders->shipping:lost<=0", "state", "unmet", "value", "11 lost")))), + "note", "the seda hand-off has no error handling")); + JsonObject held = new JsonObject( + Map.of("number", 2, "verdict", "PASS", "open", false, + "rules", new JsonArray(List.of("r3")), "ruleTexts", new JsonArray(List.of("route=refunds delay")))); + JsonObject exploration = new JsonObject( + Map.of("running", false, "finishedBy", "finished", + "experiments", new JsonArray(List.of(failed, held)))); + ChaosExplore.Run run = new ChaosExplore.Run("shop", "10m", "orders", "qwen3.6 (ollama)", -42L); + + ChaosExplore.Saved saved = ChaosExplore.save(dir, LocalDateTime.of(2026, 11, 9, 14, 2, 30), run, exploration, + "## Summary\n1 broke", List.of("Chaos exploration finished: 2 experiments, 1 failed")); + + assertThat(saved.report()).isEqualTo(dir.resolve("reports/explore-20261109-140230.md")); + assertThat(Files.readString(saved.report())).contains("# Chaos exploration: shop", + "2026-11-09 14:02 · budget 10m · focus orders · qwen3.6 (ollama) · seed -42", + "## Report\n\n## Summary\n1 broke", "Chaos exploration finished: 2 experiments, 1 failed", + "## Scenarios", "camel run * --chaos=" + dir.resolve("explore-20261109-140230-1.chaos")); + JsonObject journal = (JsonObject) Jsoner.deserialize(Files.readString(saved.journal())); + assertThat(journal.getString("integration")).isEqualTo("shop"); + assertThat(journal.getLong("seed")).isEqualTo(-42); + assertThat(journal.get("exploration")).isInstanceOf(Map.class); + // only the experiment that failed is a scenario: comments, the seed, and its rules together + assertThat(saved.scenarios()).containsExactly(dir.resolve("explore-20261109-140230-1.chaos")); + assertThat(Files.readString(saved.scenarios().get(0))).isEqualTo( + "# From the chaos exploration of shop on 2026-11-09 14:02, experiment 1: FAIL\n" + + "# hypothesis: nothing is lost after the hand-off\n" + + "# expect orders->shipping:lost<=0: unmet, 11 lost\n" + + "# note: the seda hand-off has no error handling\n" + + "# replay: camel run * --chaos=" + + dir.resolve("explore-20261109-140230-1.chaos") + + " (and send it messages as the experiment did," + + " when its routes do not feed themselves)\n" + + "seed=-42\n" + + "route=shipping error rate=0.2 until=30s" + + " expect=orders->shipping:lost<=0 within=45s\n" + + "endpoint=seda:dlq error at=dlq until=30s\n"); + } + + @Test + void saveIsOnByDefault() { + ChaosExplore command = new ChaosExplore(new CamelJBangMain()); + CommandLine.populateCommand(command); + assertThat(command.save).isTrue(); + CommandLine.populateCommand(command, "--no-save"); + assertThat(command.save).isFalse(); + } } diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/resources/golden/cli-help.txt b/dsl/camel-jbang/camel-jbang-core/src/test/resources/golden/cli-help.txt index db328bcb784c..94d33a3c2fd4 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/resources/golden/cli-help.txt +++ b/dsl/camel-jbang/camel-jbang-core/src/test/resources/golden/cli-help.txt @@ -827,9 +827,10 @@ failed<=0 within=70s' \ camel cmd chaos-experiment --end --note='stopped by hand' == camel cmd chaos-explore --help -Usage: camel cmd chaos-explore [-h] [--show-tools] [--verbose] [--yes] - [--api-key=<apiKey>] [--api-type=<apiType>] - [--budget=<budget>] [--focus=<focus>] +Usage: camel cmd chaos-explore [-h] [--[no-]save] [--show-tools] [--verbose] + [--yes] [--api-key=<apiKey>] + [--api-type=<apiType>] [--budget=<budget>] + [--focus=<focus>] [--max-iterations=<maxIterations>] [--model=<model>] [--timeout=<timeout>] [--tools=<tools>] [--url=<url>] [<name>] @@ -867,6 +868,10 @@ with faults, for a time budget, and reports what held and what broke calls that camel ask asks for first, without asking --show-tools Show the tool calls and results as they happen + --[no-]save Save the report in test/chaos/reports (as + markdown, and the journal as JSON), and each + experiment that failed as a scenario in + test/chaos, to replay it --verbose Print debug information of the LLM calls This command starts a chaos exploration with the budget (camel cmd
