This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26786-chaos-tools in repository https://gitbox.apache.org/repos/asf/camel.git
commit aea969ed8748a6bb4572d8f5c6b3ea5e9c1b993f Author: Claus Ibsen <[email protected]> AuthorDate: Mon Nov 9 16:39:15 2026 +0100 CAMEL-26786: camel-jbang - chaos exploration tools for camel ask and the MCP server: explore start, one experiment per call, explore finish Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../modules/ROOT/pages/camel-jbang-mcp.adoc | 24 +++- .../camel/dsl/jbang/core/commands/AskTools.java | 13 +- .../dsl/jbang/core/commands/ai/ToolConfirms.java | 19 ++- .../dsl/jbang/core/commands/ai/ToolGroups.java | 11 +- .../dsl/jbang/core/commands/ai/ToolRegistry.java | 116 +++++++++++++++++ .../dsl/jbang/core/commands/AskToolsTest.java | 23 +++- .../core/commands/ai/ChaosExploreToolsTest.java | 141 +++++++++++++++++++++ .../dsl/jbang/core/commands/ai/ToolGroupsTest.java | 10 +- .../jbang/core/commands/ai/ToolRegistryTest.java | 17 ++- .../dsl/jbang/core/commands/mcp/RuntimeTools.java | 84 ++++++++++++ .../mcp/RuntimeToolsChaosSecurityTest.java | 23 +++- 11 files changed, 461 insertions(+), 20 deletions(-) diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc index bf85f8735035..5b5d95819925 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc @@ -1124,14 +1124,34 @@ that changes rules is refused. The list and the report read the chaos console fr | Loads a scenario (`scenario`, the text of a `.chaos` file, and optionally its name `scenarioName`), as `camel cmd chaos-load` does: the rules there are now are removed and the report starts again, then the rules of the scenario are added, all or none. Returns the ids, each rule in plain words, and `warnings`. + +| `camel_runtime_chaos_explore_start` +| `false` | `false` +| Starts an exploration, as `camel cmd chaos-explore-start` does: a time to try rules in (`budget`, 10 minutes by + default and at most 2 hours, and an optional `focus`). While it runs a rule ends at the deadline at the latest, and at + the deadline the rules are removed, whatever the client does. + +| `camel_runtime_chaos_experiment` +| `false` | `true` +| Runs one experiment in the exploration in one call, as `camel cmd chaos-experiment` does: adds the rules (`rule`, + one of them with `expect=`), sends the messages of `feed` (with `body`, `headers`, `repeat` and `interval`), waits until + the expectations are decided (at most `wait`, 2 minutes by default), and ends it. Returns the `verdict` (`FAIL`, + `PASS` or `INCONCLUSIVE`), the expectations, the faults per rule, what the routes did, the `traffic` and `timeLeft`. + +| `camel_runtime_chaos_explore_finish` +| `false` | `false` +| Finishes the exploration before its deadline, as `camel cmd chaos-explore-finish` does, and returns its journal: the + rules tried and the experiments with their verdicts. |=== -Adding and changing a rule, and loading a scenario, are marked destructive: a rule makes messages fail, wait or get lost, and its +Adding and changing a rule, loading a scenario, and an experiment are marked destructive: a rule makes messages fail, wait or get lost, and its `exception=` can name any exception class. So a client that honours `destructiveHint` asks before it runs them, and the `read-write` access level of the security layer (`camel.mcp.security.access-level`) does not offer them, as it does not offer `camel_runtime_route_control` or `camel_runtime_sql`; `read-only` offers the list, the report, what to try and the check. Removing and clearing only take faults away, and the report keeps the removed rules, so they are not destructive. `camel ask` -has the same tools (`chaos_add` and the others), and asks the user before it adds or changes a rule or loads a scenario (showing the rules of the scenario). The +has the same tools (`chaos_add` and the others), and asks the user before it adds or changes a rule or loads a scenario (showing the rules of the scenario). It asks +once for an exploration (`chaos_explore_start`), and not again for the experiments in it, as camel-chaos removes their +rules at its deadline; the messages of an experiment are asked as `send_message` asks them. The development MCP tools that `camel run --mcp` serves over HTTP have only the list and the report: a rule is never added, changed, removed, checked, suggested or loaded over the HTTP server of the application. diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/AskTools.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/AskTools.java index 36c12a883903..f212812aa219 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/AskTools.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/AskTools.java @@ -77,7 +77,10 @@ public class AskTools { Map.entry(ToolGroups.CHAOS_REMOVE_TOOL, "chaos_remove"), Map.entry(ToolGroups.CHAOS_CLEAR_TOOL, "chaos_clear"), Map.entry(ToolGroups.CHAOS_SUGGEST_TOOL, "chaos_suggest"), - Map.entry(ToolGroups.CHAOS_LOAD_TOOL, "chaos_load")); + Map.entry(ToolGroups.CHAOS_LOAD_TOOL, "chaos_load"), + Map.entry(ToolGroups.CHAOS_EXPLORE_START_TOOL, "chaos_explore_start"), + Map.entry(ToolGroups.CHAOS_EXPERIMENT_TOOL, "chaos_experiment"), + Map.entry(ToolGroups.CHAOS_EXPLORE_FINISH_TOOL, "chaos_explore_finish")); private static final String CLI_LIST_COMMANDS = "cli_list_commands"; private static final String CLI_COMMAND_HELP = "cli_command_help"; @@ -358,6 +361,14 @@ public class AskTools { args.get("name") != null ? args.get("name") : processName(), args.get("id"), args.get("rule")); case "chaos_load" -> ToolConfirms.chaosLoad(args.get("name") != null ? args.get("name") : processName(), args.get("scenarioName"), args.get("scenario")); + // the consent to the experiments of the exploration, which camel-chaos ends at its deadline + case "chaos_explore_start" -> ToolConfirms.chaosExplore( + args.get("name") != null ? args.get("name") : processName(), args.get("budget"), args.get("focus")); + // its rules are covered by the exploration; its messages are confirmed as send_message confirms them + case "chaos_experiment" -> args.get("feed") == null || args.get("feed").isBlank() + ? null + : ToolConfirms.sendMessage(args.get("feed"), endpointUris(), + ToolConfirms.fileOutsideProject(args.get("body"), runtimeDirectory(), ctx.defaultDirectory())); default -> null; }; } diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolConfirms.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolConfirms.java index 11941657f920..db5d2b148f1b 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolConfirms.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolConfirms.java @@ -45,7 +45,8 @@ import org.apache.camel.util.URISupport; * <li>a SQL statement that is not a single SELECT (or SHOW, DESCRIBE), and a row update;</li> * <li>an expression in a language that runs code, such as Groovy;</li> * <li>a chaos rule added or changed (camel-chaos): it makes messages of the integration fail, wait or get lost, and can - * name any exception class. Removing a rule only takes faults away.</li> + * name any exception class. Removing a rule only takes faults away. A chaos exploration is confirmed once, when it + * starts: the experiments in it add rules without asking again, and camel-chaos removes them at its deadline.</li> * </ul> * A file write is confirmed with its diff ({@link #write}). {@code camel ask}, which shows no screen of the integration * while it works, also asks before a call that stops or starts an integration, browses an endpoint the integration does @@ -331,6 +332,22 @@ public final class ToolConfirms { + " they are active, the integration's messages fail, wait or get lost"); } + /** + * The call of starting a chaos exploration (chaos_explore_start): the consent to the experiments the AI then runs + * in it (chaos_experiment), each adding chaos rules without asking again. camel-chaos ends them at the deadline of + * the exploration, whatever the AI does. + */ + public static Call chaosExplore(String integration, String budget, String focus) { + String where = integration != null && !integration.isBlank() ? integration : "the selected integration"; + String time = budget != null && !budget.isBlank() ? budget.strip() : "10m"; + return new Call( + "explore chaos in " + where + " for " + time, + focus != null && !focus.isBlank() ? "focus: " + focus.strip() : "all routes", + "For that time the AI adds chaos rules without asking again, and the integration's messages fail, wait" + + " or get lost; at the end the" + + " rules are removed"); + } + /** * Splits a text with several rules separated by {@code ;}, as in camel.chaos.rules; a {@code ;} inside quotes does * not split. diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroups.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroups.java index 76824c582e0f..29aeccd83d23 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroups.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroups.java @@ -47,6 +47,9 @@ public final class ToolGroups { public static final String CHAOS_CLEAR_TOOL = "camel_runtime_chaos_clear"; public static final String CHAOS_SUGGEST_TOOL = "camel_runtime_chaos_suggest"; public static final String CHAOS_LOAD_TOOL = "camel_runtime_chaos_load"; + public static final String CHAOS_EXPLORE_START_TOOL = "camel_runtime_chaos_explore_start"; + public static final String CHAOS_EXPERIMENT_TOOL = "camel_runtime_chaos_experiment"; + public static final String CHAOS_EXPLORE_FINISH_TOOL = "camel_runtime_chaos_explore_finish"; /** * A loaded group. @@ -143,7 +146,8 @@ public final class ToolGroups { f.chaosOn() ? List.of(CHAOS_LIST_TOOL, CHAOS_REPORT_TOOL, CHAOS_SUGGEST_TOOL, CHAOS_CHECK_TOOL, CHAOS_ADD_TOOL, CHAOS_CHANGE_TOOL, CHAOS_REMOVE_TOOL, CHAOS_CLEAR_TOOL, - CHAOS_LOAD_TOOL) + CHAOS_LOAD_TOOL, CHAOS_EXPLORE_START_TOOL, CHAOS_EXPERIMENT_TOOL, + CHAOS_EXPLORE_FINISH_TOOL) : List.of(CHAOS_LIST_TOOL, CHAOS_REPORT_TOOL), describeChaos(f))); } @@ -242,7 +246,10 @@ public final class ToolGroups { + CHAOS_REPORT_TOOL + " show the rules and what they did; " + CHAOS_ADD_TOOL + " takes a rule such as endpoint=kafka:orders* error rate=0.1 until=5m, and " + CHAOS_CHECK_TOOL + " says what it would hit without adding it; " + CHAOS_SUGGEST_TOOL - + " suggests rules from what the routes do, and " + CHAOS_LOAD_TOOL + " loads a scenario."; + + " suggests rules from what the routes do, and " + CHAOS_LOAD_TOOL + " loads a scenario; " + + CHAOS_EXPLORE_START_TOOL + " starts an exploration with a time budget, in which " + CHAOS_EXPERIMENT_TOOL + + " runs one experiment per call (rules, traffic and the wait for the verdict), until " + + CHAOS_EXPLORE_FINISH_TOOL + "."; } /** {@code Circuit breakers in routes a, b}, or without the routes when they are not known. */ diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java index 16c490f78347..0c9f44d0454f 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java @@ -24,11 +24,13 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.camel.catalog.CamelCatalog; import org.apache.camel.cli.connector.RepeatedSend; import org.apache.camel.dsl.jbang.core.commands.action.CamelRouteStructureAction; +import org.apache.camel.dsl.jbang.core.common.ChaosExperimentRun; import org.apache.camel.dsl.jbang.core.common.CliException; import org.apache.camel.dsl.jbang.core.common.ExampleHelper; import org.apache.camel.dsl.jbang.core.common.ProcessFinder; @@ -928,6 +930,120 @@ public final class ToolRegistry { } }); })); + // an exploration (CAMEL-26773, CAMEL-26786): camel-chaos keeps its journal and ends its rules at the deadline, + // so starting one is the consent to the experiments in it, which need no confirm of their own + register(tool("chaos_explore_start", + "Start a chaos exploration in the integration (as camel cmd chaos-explore-start): a time to try chaos " + + "rules in, that camel-chaos enforces. While it runs, a rule ends at the " + + "deadline at the latest, and at the deadline the rules are removed, " + + "whatever happens. Its journal (the baseline, the rules tried and the " + + "experiments) is in chaos_report. Run experiments in it with " + + "chaos_experiment, and finish it with chaos_explore_finish. One runs at a " + + "time.") + .param("budget", "string", "How long it runs, between 10s and 2h (default 10m)", false) + .param("focus", "string", "What it is about, such as a route (optional)", false) + .param("name", "string", CHAOS_NAME, false) + .readOnly(false).destructive(false) + .executor((ctx, args) -> { + selectChaosProcess(ctx, args); + return ctx.executeAction("chaos", root -> { + root.put("command", "explore-start"); + putIfSet(root, "budget", args.get("budget")); + putIfSet(root, "focus", args.get("focus")); + }); + })); + + register(tool("chaos_experiment", + "Run one chaos experiment in the running exploration (as camel cmd chaos-experiment), in one call: it " + + "adds the rules (one of them with an expectation, expect=, that decides the " + + "hypothesis), sends the messages of feed when given (as send_message, with " + + "body, headers, repeat and interval), waits until the expectations are " + + "decided or wait is up, and ends the experiment, which removes its rules. " + + "Returns the verdict (FAIL: an expectation is unmet; PASS: all are met; " + + "INCONCLUSIVE: the wait was up before one was decided, so wait longer than " + + "its within=), the expectations with their actual values, the faults per " + + "rule, what the routes did meanwhile, the traffic sent, and the time left " + + "of the exploration (timeLeft, millis). chaos_suggest has rules to try and " + + "the feed of each route.") + .param("rule", "string", + "The rules in their one-line text form, separated by ;, one of them with expect=, such as" + + " 'route=refunds error rate=every:5 expect=refunds:failed<=0 within=30s'", + true) + .param("hypothesis", "string", "What the experiment tests, in a few words", false) + .param("feed", "string", + "The endpoint, or the id of the route, to send the messages to (as the feed of chaos_suggest);" + + " leave it out when the route feeds itself (a timer)", + false) + .param("body", "string", "The body of the messages (file:<path>, or file:<folder> for a message per file)", + false) + .param("headers", "string", "The headers of the messages as key=value pairs separated by newlines", false) + .param("repeat", "string", "How many messages to send in all (at most 10000)", false) + .param("interval", "string", + "The pause between two messages in millis, or a range such as 100..500 (at most 60000)", false) + .param("wait", "string", + "The longest wait for the verdict after the messages were sent, such as 90s (default 2m, at most" + + " 10m)", + false) + .param("note", "string", "What to note with the experiment in the journal", false) + .param("name", "string", CHAOS_NAME, false) + // its rules make messages fail, wait or get lost, within the exploration the user started + .readOnly(false).destructive(true) + .executor((ctx, args) -> { + String rule = AuthoringTools.required(args, "rule"); + String feed = args.get("feed"); + long wait; + try { + RepeatedSend.interval(args.get("interval")); + wait = ChaosExperimentRun.waitMillis(args.get("wait")); + } catch (IllegalArgumentException e) { + throw new ToolExecutionException(e.getMessage()); + } + int repeat = intArg(args, "repeat", 0); + selectChaosProcess(ctx, args); + ChaosExperimentRun.Feed traffic = feed != null && !feed.isBlank() + ? new ChaosExperimentRun.Feed( + feed, args.get("body"), args.get("headers"), repeat, + args.get("interval")) + : null; + return ChaosExperimentRun.run(new ChaosExperimentRun.Integration() { + @Override + public JsonObject chaos(String command, Consumer<JsonObject> options) { + String answer = ctx.executeAction("chaos", root -> { + root.put("command", command); + options.accept(root); + }); + return Jsoner.deserialize(answer, new JsonObject()); + } + + @Override + public JsonObject send(ChaosExperimentRun.Feed f) { + return ctx.sendMessage(f.endpoint(), f.body(), f.headers(), f.repeat(), f.interval()); + } + + @Override + public JsonObject status() { + return ctx.readFullStatus(); + } + }, args.get("hypothesis"), rule, traffic, wait, args.get("note")).toJson(); + })); + + register(tool("chaos_explore_finish", + "Finish the chaos exploration of the integration before its deadline (as camel cmd chaos-explore-finish):" + + " the rules are removed (an open experiment is ended), and the journal " + + "stays in chaos_report. Returns the journal: the rules tried, the " + + "experiments with their verdicts, and the baseline.") + .param("name", "string", CHAOS_NAME, false) + .readOnly(false).destructive(false) + .executor((ctx, args) -> { + selectChaosProcess(ctx, args); + return chaos(ctx, "explore-finish", null, null); + })); + } + + private static void putIfSet(JsonObject root, String key, String value) { + if (value != null && !value.isBlank()) { + root.put(key, value); + } } /** diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskToolsTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskToolsTest.java index 0020ea06cf3e..1d883f794194 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskToolsTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskToolsTest.java @@ -238,11 +238,32 @@ class AskToolsTest { // the user sees the rules of the scenario assertThat(load.target()).isEqualTo("# outage\nendpoint=http:* error"); for (String name : List.of("chaos_check", "chaos_suggest", "chaos_remove", "chaos_clear", "chaos_list", - "chaos_report")) { + "chaos_report", "chaos_explore_finish")) { assertThat(tools.confirmation(name, Map.of("id", "r1"))).as(name).isNull(); } } + @Test + void anExplorationIsAskedOnceAndItsExperimentsNot() { + AskTools tools = new AskTools(-1, call -> true); + + // the consent to the experiments in it, which camel-chaos ends at its deadline + ToolConfirms.Call explore = tools.confirmation("chaos_explore_start", + Map.of("name", "shop", "budget", "30m", "focus", "refunds")); + assertThat(explore.title()).isEqualTo("explore chaos in shop for 30m"); + assertThat(explore.target()).isEqualTo("focus: refunds"); + assertThat(tools.confirmation("chaos_explore_start", Map.of()).title()) + .isEqualTo("explore chaos in the selected integration for 10m"); + // an experiment adds its rules without asking, and its messages to a route stay inside the integration + assertThat(tools.confirmation("chaos_experiment", + Map.of("rule", "route=refunds error expect=refunds:failed<=0", "feed", "refunds"))).isNull(); + assertThat(tools.confirmation("chaos_experiment", Map.of("rule", "route=refunds error"))).isNull(); + // a message that leaves the integration is asked, as send_message asks it + assertThat(tools.confirmation("chaos_experiment", + Map.of("rule", "route=refunds error", "feed", "kafka:refunds")).title()) + .isEqualTo("send a message to an endpoint the integration does not have"); + } + @Test void theChaosGroupNamesTheSharedTools() { AppFeatures chaos = AppFeatures.fromStatus(new JsonObject(Map.of("chaos", Map.of("enabled", true)))); diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExploreToolsTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExploreToolsTest.java new file mode 100644 index 000000000000..ef6ff23e408c --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ChaosExploreToolsTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.ai; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.camel.dsl.jbang.core.common.CommandLineHelper; +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The tools of a chaos exploration (CAMEL-26786) against an integration that answers their actions: the exploration + * starts and finishes, and an experiment runs in one call, with its traffic and the wait for its verdict in the status. + */ +class ChaosExploreToolsTest { + + private static final long PID = 4713L; + + @TempDir + Path home; + + private Path previousHome; + private FakeIntegration integration; + + @BeforeEach + void start() throws Exception { + previousHome = CommandLineHelper.getHomeDir(); + CommandLineHelper.useHomeDir(home.toString()); + Files.createDirectories(CommandLineHelper.getCamelDir()); + integration = new FakeIntegration(PID, ChaosExploreToolsTest::answer); + } + + @AfterEach + void stop() throws Exception { + integration.close(); + CommandLineHelper.useHomeDir(previousHome.toString()); + } + + /** The answers of an integration with chaos on, to the chaos commands and the send. */ + private static Object answer(JsonObject action) { + if ("send".equals(action.getString("action"))) { + return Map.of("status", "failed", "endpoint", "direct://refunds", "messages", 10, "sent", 10, "failed", 2); + } + return switch (action.getString("command")) { + case "explore-start" -> Map.of("running", true, "budget", 300_000, "focus", "refunds"); + case "explore-finish" -> Map.of("running", false, "finishedBy", "finished", "rules", List.of("r1")); + case "experiment-begin" -> experiment("PENDING", true); + case "experiment-end" -> experiment("FAIL", false); + default -> Map.of("status", "failed", "error", "Unknown chaos command"); + }; + } + + private static Map<String, Object> experiment(String verdict, boolean open) { + return Map.of("number", 1, "hypothesis", "a refund never fails", "verdict", verdict, "open", open, + "rules", List.of("r1"), "expectations", + List.of(Map.of("expect", "refunds:failed<=0", "state", open ? "pending" : "unmet", "actual", 2))); + } + + private static Object call(String tool, Map<String, String> args) { + ToolContext ctx = new ToolContext(); + ctx.selectProcess(PID); + return ToolRegistry.execute(tool, ctx, new HashMap<>(args)); + } + + private JsonObject lastChaosCommand(String command) { + return integration.actions().stream().filter(a -> command.equals(a.getString("command"))).reduce((a, b) -> b) + .orElseThrow(); + } + + @Test + void anExplorationStartsAndFinishes() throws Exception { + String started = call("chaos_explore_start", Map.of("budget", "5m", "focus", "refunds")).toString(); + + assertThat(started).contains("\"running\":true"); + JsonObject action = lastChaosCommand("explore-start"); + assertThat(action.getString("action")).isEqualTo("chaos"); + assertThat(action.getString("budget")).isEqualTo("5m"); + assertThat(action.getString("focus")).isEqualTo("refunds"); + + assertThat(call("chaos_explore_finish", Map.of()).toString()).contains("\"finishedBy\":\"finished\""); + } + + @Test + void anExperimentInOneCall() throws Exception { + // the status shows the verdict decided: the wait ends at the first look + JsonObject exploration = new JsonObject( + Map.of("running", true, "timeLeft", 240_000, + "experiments", List.of(experiment("FAIL", true)))); + JsonObject status = new JsonObject( + Map.of("chaos", + Map.of("enabled", true, "report", Map.of("exploration", exploration)))); + Files.writeString(CommandLineHelper.getCamelDir().resolve(PID + "-status.json"), status.toJson()); + + String answer = call("chaos_experiment", Map.of( + "hypothesis", "a refund never fails", + "rule", "route=refunds error rate=every:5 expect=refunds:failed<=0 within=20s", + "feed", "refunds", "body", "R-1", "repeat", "10", "interval", "100ms", + "note", "no error handler")).toString(); + + JsonObject result = (JsonObject) Jsoner.deserialize(answer); + assertThat(result.getString("verdict")).isEqualTo("FAIL"); + assertThat(result.getLong("timeLeft")).isEqualTo(240_000); + assertThat(((JsonObject) result.get("traffic")).getLong("sent")).isEqualTo(10); + // begin, the messages, and the end with the note + assertThat(integration.actions()).extracting(a -> a.getString("command") != null + ? a.getString("command") : a.getString("action")) + .containsExactly("experiment-begin", "send", "experiment-end"); + JsonObject begin = lastChaosCommand("experiment-begin"); + assertThat(begin.getString("hypothesis")).isEqualTo("a refund never fails"); + assertThat(begin.getString("rule")).startsWith("route=refunds error"); + JsonObject send = integration.actions().get(1); + assertThat(send.getString("endpoint")).isEqualTo("refunds"); + assertThat(send.getLong("repeat")).isEqualTo(10); + assertThat(send.getLong("interval")).isEqualTo(100); + assertThat(lastChaosCommand("experiment-end").getString("note")).isEqualTo("no error handler"); + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroupsTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroupsTest.java index 6b9776a79d95..bc995052c660 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroupsTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolGroupsTest.java @@ -144,18 +144,22 @@ class ToolGroupsTest { } @Test - void chaosLoadsItsNineToolsWhenOnAndOnlyTheReadsWhenOff() { + void chaosLoadsItsTwelveToolsWhenOnAndOnlyTheReadsWhenOff() { AppFeatures on = AppFeatures.fromStatus(new JsonObject(Map.of("chaos", Map.of("enabled", true)))); ToolGroups.Selection s = ToolGroups.select(on); assertEquals(List.of(ToolGroup.CHAOS), s.toolGroups()); assertEquals(List.of("camel_runtime_chaos_list", "camel_runtime_chaos_report", "camel_runtime_chaos_suggest", "camel_runtime_chaos_check", "camel_runtime_chaos_add", "camel_runtime_chaos_change", - "camel_runtime_chaos_remove", "camel_runtime_chaos_clear", "camel_runtime_chaos_load"), s.mcpTools()); + "camel_runtime_chaos_remove", "camel_runtime_chaos_clear", "camel_runtime_chaos_load", + "camel_runtime_chaos_explore_start", "camel_runtime_chaos_experiment", + "camel_runtime_chaos_explore_finish"), s.mcpTools()); assertEquals("Chaos: camel-chaos is on, so failures may be injected on purpose; camel_runtime_chaos_list and" + " camel_runtime_chaos_report show the rules and what they did; camel_runtime_chaos_add takes a rule" + " such as endpoint=kafka:orders* error rate=0.1 until=5m, and camel_runtime_chaos_check says what it" + " would hit without adding it; camel_runtime_chaos_suggest suggests rules from what the routes do," - + " and camel_runtime_chaos_load loads a scenario.", + + " and camel_runtime_chaos_load loads a scenario; camel_runtime_chaos_explore_start starts an" + + " exploration with a time budget, in which camel_runtime_chaos_experiment runs one experiment per" + + " call (rules, traffic and the wait for the verdict), until camel_runtime_chaos_explore_finish.", s.guidance().get(0)); assertEquals("chaos|", s.fingerprint()); diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistryTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistryTest.java index 8fa9bb960510..00fcce3a0d7d 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistryTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistryTest.java @@ -321,12 +321,12 @@ class ToolRegistryTest { assertTrue(t.isReadOnly(), name); assertFalse(t.isDestructive(), name); } - for (String name : List.of("chaos_add", "chaos_change", "chaos_load")) { + for (String name : List.of("chaos_add", "chaos_change", "chaos_load", "chaos_experiment")) { ToolDescriptor t = ToolRegistry.findTool(name); assertFalse(t.isReadOnly(), name); assertTrue(t.isDestructive(), name); } - for (String name : List.of("chaos_remove", "chaos_clear")) { + for (String name : List.of("chaos_remove", "chaos_clear", "chaos_explore_start", "chaos_explore_finish")) { ToolDescriptor t = ToolRegistry.findTool(name); assertFalse(t.isReadOnly(), name); assertFalse(t.isDestructive(), name); @@ -349,6 +349,17 @@ class ToolRegistryTest { assertEquals("scenario is required", assertThrows(ToolExecutionException.class, () -> ToolRegistry.execute("chaos_load", new ToolContext(), Map.of("scenarioName", "outage"))) .getMessage()); + assertEquals("rule is required", assertThrows(ToolExecutionException.class, + () -> ToolRegistry.execute("chaos_experiment", new ToolContext(), Map.of("feed", "orders"))) + .getMessage()); + assertEquals("The wait must be at most 10m, was: 1h", assertThrows(ToolExecutionException.class, + () -> ToolRegistry.execute("chaos_experiment", new ToolContext(), + Map.of("rule", "route=orders error expect=orders:failed<=0", "wait", "1h"))) + .getMessage()); + assertTrue(assertThrows(ToolExecutionException.class, + () -> ToolRegistry.execute("chaos_experiment", new ToolContext(), + Map.of("rule", "route=orders error expect=orders:failed<=0", "interval", "500..100"))) + .getMessage().contains("from the shortest to the longest")); } @Test @@ -388,7 +399,7 @@ class ToolRegistryTest { // only the chaos tools that change the rules, the check (the integration parses the rule) and what to try (an // action of the CLI connector) stay off HTTP; the other tools keep what they offered assertEquals(List.of("chaos_add", "chaos_check", "chaos_suggest", "chaos_load", "chaos_change", "chaos_remove", - "chaos_clear"), left); + "chaos_clear", "chaos_explore_start", "chaos_experiment", "chaos_explore_finish"), left); assertTrue(http.containsAll(List.of("chaos_list", "chaos_report", "stop_route", "execute_sql"))); } } diff --git a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java index 1f7f8e290b26..95e1d96f308f 100644 --- a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java +++ b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java @@ -1266,6 +1266,90 @@ public class RuntimeTools { Boolean.TRUE.equals(reset) ? Map.of("reset", "true") : Map.of()); } + @ToolGuardrails(input = McpArgumentGuardrail.class, output = McpArgumentGuardrail.class) + @Tool(annotations = @Tool.Annotations(readOnlyHint = false, destructiveHint = false, openWorldHint = false), + description = """ + Start a chaos exploration in the Camel application (as camel cmd chaos-explore-start): a time to \ + try chaos rules in, that camel-chaos enforces. While it runs, a rule ends at the deadline at the \ + latest, and at the deadline the rules are removed, whatever happens. Its journal (the baseline, \ + the rules tried and the experiments) is in camel_runtime_chaos_report. Run experiments in it with \ + camel_runtime_chaos_experiment, and finish it with camel_runtime_chaos_explore_finish. One runs \ + at a time.""") + public JsonObject camel_runtime_chaos_explore_start( + @ToolArg(description = CHAOS_NAME_OR_PID_DESC, required = false) String nameOrPid, + @ToolArg(description = "How long it runs, between 10s and 2h (default 10m)", required = false) String budget, + @ToolArg(description = "What it is about, such as a route", required = false) String focus) { + Map<String, String> args = new HashMap<>(); + putIfSet(args, "budget", budget); + putIfSet(args, "focus", focus); + return delegateChaos("chaos_explore_start", nameOrPid, args); + } + + @ToolGuardrails(input = McpArgumentGuardrail.class, output = McpArgumentGuardrail.class) + @Tool(annotations = @Tool.Annotations(readOnlyHint = false, destructiveHint = true, openWorldHint = false), + description = """ + Run one chaos experiment in the running exploration of the Camel application (as camel cmd \ + chaos-experiment), in one call: it adds the rules (one of them with an expectation, expect=, that \ + decides the hypothesis), sends the messages of feed when given (as camel_runtime_send, with body, \ + headers, repeat and interval), waits until the expectations are decided or wait is up, and ends \ + the experiment, which removes its rules. Returns the verdict (FAIL: an expectation is unmet; PASS: \ + all are met; INCONCLUSIVE: the wait was up before one was decided, so wait longer than its \ + within=), the expectations with their actual values, the faults per rule, what the routes did \ + meanwhile, the traffic sent, and the time left of the exploration (timeLeft, millis). \ + camel_runtime_chaos_suggest has rules to try and the feed of each route.""") + public JsonObject camel_runtime_chaos_experiment( + @ToolArg(description = CHAOS_NAME_OR_PID_DESC, required = false) String nameOrPid, + @ToolArg(description = "The rules in their one-line text form, separated by ;, one of them with expect=," + + " such as 'route=refunds error rate=every:5 expect=refunds:failed<=0 within=30s'") String rule, + @ToolArg(description = "What the experiment tests, in a few words", required = false) String hypothesis, + @ToolArg(description = "The endpoint, or the id of the route, to send the messages to (as the feed of" + + " camel_runtime_chaos_suggest); leave it out when the route feeds itself", + required = false) String feed, + @ToolArg(description = "The body of the messages (file:<path>, or file:<folder> for a message per file)", + required = false) String body, + @ToolArg(description = "The headers of the messages as key=value pairs separated by newlines", + required = false) String headers, + @ToolArg(description = "How many messages to send in all (at most 10000)", required = false) Integer repeat, + @ToolArg(description = "The pause between two messages in millis, or a range such as 100..500", + required = false) String interval, + @ToolArg(description = "The longest wait for the verdict after the messages were sent, such as 90s" + + " (default 2m, at most 10m)", + required = false) String wait, + @ToolArg(description = "What to note with the experiment in the journal", required = false) String note) { + requireArg("rule", rule); + Map<String, String> args = new HashMap<>(); + args.put("rule", rule); + putIfSet(args, "hypothesis", hypothesis); + putIfSet(args, "feed", feed); + putIfSet(args, "body", body); + putIfSet(args, "headers", headers); + if (repeat != null) { + args.put("repeat", repeat.toString()); + } + putIfSet(args, "interval", interval); + putIfSet(args, "wait", wait); + putIfSet(args, "note", note); + return delegateChaos("chaos_experiment", nameOrPid, args); + } + + @ToolGuardrails(input = McpArgumentGuardrail.class, output = McpArgumentGuardrail.class) + @Tool(annotations = @Tool.Annotations(readOnlyHint = false, destructiveHint = false, openWorldHint = false), + description = """ + Finish the chaos exploration of the Camel application before its deadline (as camel cmd \ + chaos-explore-finish): the rules are removed (an open experiment is ended), and the journal stays \ + in camel_runtime_chaos_report. Returns the journal: the rules tried, the experiments with their \ + verdicts, and the baseline.""") + public JsonObject camel_runtime_chaos_explore_finish( + @ToolArg(description = CHAOS_NAME_OR_PID_DESC, required = false) String nameOrPid) { + return delegateChaos("chaos_explore_finish", nameOrPid, Map.of()); + } + + private static void putIfSet(Map<String, String> args, String key, String value) { + if (value != null && !value.isBlank()) { + args.put(key, value); + } + } + /** * As {@link #delegateToRegistry(String, String, Map)}, for a chaos tool: without a name, the only integration * running, else the only one with chaos switched on ({@link RuntimeService#findChaosProcess(String)}). diff --git a/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosSecurityTest.java b/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosSecurityTest.java index 510069c74919..2c5bbd2b0468 100644 --- a/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosSecurityTest.java +++ b/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosSecurityTest.java @@ -34,8 +34,9 @@ import static org.assertj.core.api.Assertions.assertThat; /** * The marks of the chaos tools and what they mean: adding and changing a rule is destructive and asked first - * ({@link ToolConfirms}), the list and the report only read, and each access level of the security layer - * ({@link McpAccessFilter} decides from the annotations) offers the tools by their marks. + * ({@link ToolConfirms}), an experiment is asked once with the start of its exploration, the list and the report only + * read, and each access level of the security layer ({@link McpAccessFilter} decides from the annotations) offers the + * tools by their marks. */ class RuntimeToolsChaosSecurityTest { @@ -52,6 +53,10 @@ class RuntimeToolsChaosSecurityTest { MARKS.put("camel_runtime_chaos_remove", new boolean[] { false, false }); MARKS.put("camel_runtime_chaos_clear", new boolean[] { false, false }); MARKS.put("camel_runtime_chaos_load", new boolean[] { false, true }); + // an exploration bounds the faults by its deadline; an experiment adds rules, so it is destructive + MARKS.put("camel_runtime_chaos_explore_start", new boolean[] { false, false }); + MARKS.put("camel_runtime_chaos_experiment", new boolean[] { false, true }); + MARKS.put("camel_runtime_chaos_explore_finish", new boolean[] { false, false }); } private static Tool.Annotations annotations(String name) { @@ -89,11 +94,14 @@ class RuntimeToolsChaosSecurityTest { void exactlyTheDestructiveToolsAreInToolConfirms() { MARKS.forEach((name, marks) -> { String command = name.substring("camel_runtime_chaos_".length()); - ToolConfirms.Call call = "load".equals(command) - ? ToolConfirms.chaosLoad("shop", "outage", "endpoint=kafka:* error rate=0.1") - : ToolConfirms.chaos(command, "shop", "r1", "endpoint=kafka:* error rate=0.1"); + ToolConfirms.Call call = switch (command) { + case "load" -> ToolConfirms.chaosLoad("shop", "outage", "endpoint=kafka:* error rate=0.1"); + // asked once, when its exploration starts (chaos_explore_start), for all its experiments + case "experiment" -> ToolConfirms.chaosExplore("shop", "10m", null); + default -> ToolConfirms.chaos(command, "shop", "r1", "endpoint=kafka:* error rate=0.1"); + }; assertThat(call != null).as(name + " is asked first").isEqualTo(marks[1]); - if (call != null) { + if (call != null && !"experiment".equals(command)) { // the user sees the rule, or the rules of the scenario assertThat(call.target()).isEqualTo("endpoint=kafka:* error rate=0.1"); } @@ -109,7 +117,8 @@ class RuntimeToolsChaosSecurityTest { assertThat(offered(McpSecurityConfig.AccessLevel.READ_WRITE)).containsExactly( "camel_runtime_chaos_list", "camel_runtime_chaos_report", "camel_runtime_chaos_suggest", "camel_runtime_chaos_check", - "camel_runtime_chaos_remove", "camel_runtime_chaos_clear"); + "camel_runtime_chaos_remove", "camel_runtime_chaos_clear", "camel_runtime_chaos_explore_start", + "camel_runtime_chaos_explore_finish"); assertThat(offered(McpSecurityConfig.AccessLevel.ADMIN)).containsExactlyElementsOf(MARKS.keySet()); // with the security layer on, the interceptor of @McpSecured audits each call (and redacts the answer when // camel.mcp.security.redaction.enabled=true)
