This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26725-chaos-test-with-ai in repository https://gitbox.apache.org/repos/asf/camel.git
commit 9f926f3fc8b02a864f1f5b3fd82eb16202d062ca Author: Claus Ibsen <[email protected]> AuthorDate: Sat Nov 7 23:00:32 2026 +0100 CAMEL-26725: camel-jbang - make a test of a chaos rule with AI, so a finding runs again with the same faults t on a rule of the chaos screen asks the AI panel to make a test of it, with the rule, the seed and what the rule did: a JUnit test with chaos switched on and ChaosAssertions for a Maven project, or a scenario with an expectation and the camel run --chaos=<file> --max-seconds command, whose exit code is the verdict, for a folder that runs with camel run. The AI adds an expectation when the rule has none, and writes the file with camel_write_file, which asks first. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../org/apache/camel/catalog/docs/chaos.adoc | 5 +- components/camel-chaos/src/main/docs/chaos.adoc | 5 +- .../ROOT/pages/camel-jbang-tui-observe.adoc | 5 ++ .../dsl/jbang/core/commands/tui/ChaosAiPrompt.java | 55 +++++++++++++++++++++- .../dsl/jbang/core/commands/tui/ChaosControl.java | 3 ++ .../dsl/jbang/core/commands/tui/ChaosTab.java | 6 +++ .../src/main/resources/tui/help/chaos.md | 8 ++++ .../jbang/core/commands/tui/ChaosAiPromptTest.java | 47 +++++++++++++++++- 8 files changed, 130 insertions(+), 4 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc index 7479bbdd7d16..b98acd6eed9d 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc @@ -257,7 +257,10 @@ of a scenario says which one it came from (`(scenario outage.chaos)` in the text The chaos screen of the Camel TUI saves the rules there are as a scenario (key `s`), with the seed: rules tried by hand become a file of the project, to run again. On its Errors tab, `!` makes a rule of a real error, which fails the same way again (the same exception and message at the same step, or the same HTTP status at the same endpoint, ten times): -fix the bug, then prove the fix against the same failure, and keep the rule in a scenario. +fix the bug, then prove the fix against the same failure, and keep the rule in a scenario. And `t` on a rule asks its +AI panel to make a test of it: a JUnit test with `ChaosAssertions` for a Maven project, or a scenario with an +expectation and the `camel run --chaos=<file> --max-seconds=<seconds>` command, whose exit code is the verdict, for a +folder that runs with `camel run`. == Rules diff --git a/components/camel-chaos/src/main/docs/chaos.adoc b/components/camel-chaos/src/main/docs/chaos.adoc index 7479bbdd7d16..b98acd6eed9d 100644 --- a/components/camel-chaos/src/main/docs/chaos.adoc +++ b/components/camel-chaos/src/main/docs/chaos.adoc @@ -257,7 +257,10 @@ of a scenario says which one it came from (`(scenario outage.chaos)` in the text The chaos screen of the Camel TUI saves the rules there are as a scenario (key `s`), with the seed: rules tried by hand become a file of the project, to run again. On its Errors tab, `!` makes a rule of a real error, which fails the same way again (the same exception and message at the same step, or the same HTTP status at the same endpoint, ten times): -fix the bug, then prove the fix against the same failure, and keep the rule in a scenario. +fix the bug, then prove the fix against the same failure, and keep the rule in a scenario. And `t` on a rule asks its +AI panel to make a test of it: a JUnit test with `ChaosAssertions` for a Maven project, or a scenario with an +expectation and the `camel run --chaos=<file> --max-seconds=<seconds>` command, whose exit code is the verdict, for a +folder that runs with `camel run`. == Rules diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc index 3ce16ed356b6..18ac3e5c79c4 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc @@ -338,6 +338,11 @@ fails the same way again (see <<_troubleshooting_errors,Troubleshooting Errors>> image::jbang/camel-tui-chaos-here.png[! on the to callStock of the Diagram: the known rules for its HTTP endpoint and its step, with the endpoint filled in] * *x* removes the selected rule, *c* removes all rules (the report keeps what they did), and *R* starts a new experiment: no rules and a new report, with the same seed. +* *t* makes a test of the selected rule with AI, so the finding runs again with the same faults. The AI panel opens with +the rule, the seed and what the rule did. For a Maven project the AI writes a JUnit test with chaos switched on and +`ChaosAssertions` (see the camel-chaos documentation); for a folder that runs with `camel run`, a scenario with an +expectation and the `camel run --chaos=<file> --max-seconds=<seconds>` command whose exit code is the verdict. It adds +an expectation when the rule has none, and asks before it writes the file. * *s* saves the rules as a scenario in the folder of the integration: `chaos/<name>.chaos` unless you type another file, with the seed and one rule per line, so the experiment can run again with `camel cmd chaos-load` or `camel run --chaos=<file>`. A file that is there already is replaced only when you say yes. diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPrompt.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPrompt.java index b9a0fb741d0a..70ef721e3740 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPrompt.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPrompt.java @@ -16,6 +16,8 @@ */ package org.apache.camel.dsl.jbang.core.commands.tui; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -24,7 +26,7 @@ import java.util.List; * error a rule made on the Errors tab), CAMEL-26702: the rule, its live state and the counts of the routes, and what to * do with them - say in plain words what the fault simulates and what the routes did with it, then suggest how they * should cope, and change the routes only when the user agrees. It is put in the input of the panel, for the user to - * send with Enter or change first. + * send with Enter or change first. The question to make a test of a rule ({@code t}, CAMEL-26725) is put there too. */ final class ChaosAiPrompt { @@ -48,6 +50,57 @@ final class ChaosAiPrompt { return sb.toString(); } + /** + * The question to make a test of a rule ({@code t} on the Chaos screen, CAMEL-26725), so the finding runs again + * with the same faults: the rule, the seed and what the rule did, then a JUnit test with {@code ChaosAssertions} + * for a Maven project, or a scenario with an expectation and the run whose exit code is the verdict for a folder + * that runs with {@code camel run}. The AI writes the file with the file tools of the panel, which ask first. + */ + static String ofTest(IntegrationInfo info, ChaosInfo.Rule rule) { + long seed = info.chaos != null ? info.chaos.seed : 0; + StringBuilder sb = new StringBuilder(); + sb.append("Make a test of chaos rule ").append(rule.id).append(" of the running integration ").append(name(info)) + .append(", so this finding runs again with the same faults. The rule (camel-chaos, injected on purpose):\n") + .append(" ").append(rule.text).append('\n').append("The seed of its random draws: ").append(seed) + .append(".\n"); + state(sb, info, rule); + String expectation = rule.expectRoute != null + ? "The rule has an expectation: keep it." + : "The rule has no expectation (expect=): add one that says what must hold, such as" + + " expect=<route>:<count> within=10s (at least that many exchanges complete in the route) or" + + " expect=<route>:handled<=0 (no message goes to the dead letter channel)."; + if (isMavenProject(info)) { + sb.append("The integration is a Maven project. Read its routes first, then write a JUnit test in src/test/java") + .append(" that runs them with chaos switched on: add org.apache.camel:camel-chaos as a test dependency") + .append(" when the pom.xml does not have it, and set the properties camel.chaos.enabled=true,") + .append(" camel.chaos.seed=").append(seed).append(" and camel.chaos.rules=<the rule> (CamelTestSupport:") + .append(" override configureContext and call config.withUseOverridePropertiesWithPropertiesComponent(props);") + .append(" a camel-test-main-junit6 test sets the same properties in its configuration). ") + .append(expectation) + .append(" Send the messages the test needs to the route (replace a consumer that needs a broker or a") + .append(" server with a direct endpoint, using AdviceWith), and assert with") + .append(" org.apache.camel.component.chaos.ChaosAssertions: assertFired(context, \"").append(rule.id) + .append("\", <at least>), assertExpectationMet(context, \"").append(rule.id) + .append("\", Duration.ofSeconds(10)) and assertVerdict(context, ChaosReport.PASS, Duration.ofSeconds(10))."); + } else { + sb.append("The integration runs with camel run, without a Maven project, so the test is a scenario with an") + .append(" expectation, and a run whose exit code is the verdict. Write the scenario chaos/") + .append(rule.id).append("-test.chaos: a comment line that says what it tests, seed=").append(seed) + .append(", and the rule. ").append(expectation) + .append(" Then give the command that runs it, with --max-seconds longer than the within= of the") + .append(" expectation: camel run * --chaos=chaos/").append(rule.id) + .append("-test.chaos --max-seconds=<seconds> --chaos-report=target/chaos-report.json. It ends with exit") + .append(" code 1 when the verdict is FAIL, so it can run in a pipeline."); + } + sb.append(" Write the file with camel_write_file, which asks me first, and change no route."); + return sb.toString(); + } + + /** Whether the folder of the integration is a Maven project, which can have a JUnit test. */ + static boolean isMavenProject(IntegrationInfo info) { + return info.directory != null && Files.isRegularFile(Path.of(info.directory, "pom.xml")); + } + /** The most routes and sends quoted in a request, so a big integration does not flood it. */ static final int MAX_LISTED = 20; diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java index c9d794598c3b..0b4e76b4bcf4 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java @@ -633,6 +633,9 @@ final class ChaosControl { TuiHelper.hint(spans, "R", "new experiment"); if (ctx.askAiCallback != null) { TuiHelper.hint(spans, "d", "describe with AI"); + if (hasRules) { + TuiHelper.hint(spans, "t", "make a test with AI"); + } } } } diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java index de3f9b8e3d3c..06a58059ba0f 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java @@ -99,6 +99,12 @@ class ChaosTab extends AbstractTableTab { } return true; } + if (ke.isChar('t') && rule != null && rule.id != null && ctx.askAiCallback != null + && control.step() == ChaosControl.Step.NONE) { + // a test of the rule, so the finding runs again with the same faults (CAMEL-26725) + ctx.askAiCallback.ask(ChaosAiPrompt.ofTest(info, rule)); + return true; + } return control.handleKey(ke, info, rule); } diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md index 85675d6d60fe..7e7be29d76be 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md @@ -82,6 +82,14 @@ error handler (HAND) and the redeliveries (REDEL). one rule per line. Load it again with `camel cmd chaos-load`, or start with it: `camel run --chaos=chaos/<name>.chaos`. A file that is there already is replaced only when you say yes +- **t** — Make a test of the selected rule with AI, so the finding runs + again with the same faults: the AI panel opens with a question with + the rule, the seed and what it did. For a Maven project the AI writes a + JUnit test with chaos switched on and `ChaosAssertions`; for a folder + that runs with `camel run` it writes a scenario with an expectation + (`chaos/r1-test.chaos`) and gives the `camel run --chaos=... + --max-seconds=...` command whose exit code is the verdict. It adds an + expectation when the rule has none, and asks before it writes a file - **Shift+F8** — Explain with AI: the AI panel opens with a question about the selected rule and what it did (its rate, its effect, the expectation, the routes its faults were in, the recent faults). The AI diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPromptTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPromptTest.java index c2b6f36dbe9d..c552725aa343 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPromptTest.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosAiPromptTest.java @@ -16,6 +16,7 @@ */ package org.apache.camel.dsl.jbang.core.commands.tui; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -29,12 +30,13 @@ import org.apache.camel.util.json.JsonObject; import org.apache.camel.util.json.Jsoner; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import static org.assertj.core.api.Assertions.assertThat; /** * CAMEL-26702: explain with AI (Shift+F8) what a chaos rule did, on the Chaos screen and on an error a rule made on the - * Errors tab, where it takes the place of fix with AI: the error is not a bug. + * Errors tab, where it takes the place of fix with AI: the error is not a bug. CAMEL-26725: make a test of a rule (t). */ class ChaosAiPromptTest { @@ -248,4 +250,47 @@ class ChaosAiPromptTest { spans.forEach(s -> sb.append(s.content())); return sb.toString(); } + + @Test + void aTestOfARuleForAMavenProjectIsAJUnitTestWithTheAssertions(@TempDir Path dir) throws Exception { + Files.writeString(dir.resolve("pom.xml"), "<project/>"); + info.directory = dir.toString(); + ChaosInfo.Rule rule = info.chaos.rules.get(0); + + String question = ChaosAiPrompt.ofTest(info, rule); + assertThat(question).startsWith("Make a test of chaos rule r1 of the running integration chaosdemo") + .contains("endpoint=http://localhost:8765/stock* error status=503 rate=0.2 expect=orders:handled<=0") + .contains("The seed of its random draws: 42.") + .contains("Expectation: orders:handled<=0, unmet") + .contains("a JUnit test in src/test/java").contains("camel.chaos.seed=42") + .contains("assertExpectationMet(context, \"r1\", Duration.ofSeconds(10))") + .contains("The rule has an expectation: keep it.") + .endsWith("Write the file with camel_write_file, which asks me first, and change no route."); + } + + @Test + void aTestOfARuleForACamelRunFolderIsAScenarioAndTheRun(@TempDir Path dir) throws Exception { + info.directory = dir.toString(); + ChaosInfo.Rule rule = info.chaos.rules.get(0); + rule.expectRoute = null; + + String question = ChaosAiPrompt.ofTest(info, rule); + assertThat(question).contains("runs with camel run, without a Maven project") + .contains("Write the scenario chaos/r1-test.chaos").contains("seed=42") + .contains("The rule has no expectation (expect=): add one") + .contains("camel run * --chaos=chaos/r1-test.chaos --max-seconds=<seconds>") + .doesNotContain("JUnit"); + } + + @Test + void tOnTheChaosScreenAsksTheAiToMakeATest() { + ChaosTab tab = new ChaosTab(ctx); + TuiTestHelper.renderToString(tab, 160, 40); + List<Span> footer = new ArrayList<>(); + tab.renderFooter(footer); + assertThat(footer.stream().map(Span::content).reduce("", String::concat)).contains("make a test with AI"); + + tab.handleKeyEvent(KeyEvent.ofChar('t', KeyModifiers.NONE)); + assertThat(asked).singleElement().asString().startsWith("Make a test of chaos rule r1"); + } }
