This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
The following commit(s) were added to refs/heads/main by this push:
new 91cd5e6 ai-benchmark: keep the received log in route-filter step 1,
no finished script in the data-mapping seed, and the checkpoint after a failed
step restores the seed files
91cd5e6 is described below
commit 91cd5e6c5d9d62a0c4c95804fc507542219e1833
Author: Claus Ibsen <[email protected]>
AuthorDate: Tue Oct 20 08:15:12 2026 +0200
ai-benchmark: keep the received log in route-filter step 1, no finished
script in the data-mapping seed, and the checkpoint after a failed step
restores the seed files
From the r19 analysis (Oct 20):
- route-filter-and-multicast step 1 checks for "ORD-1003 received, status
pending", but the request never said to keep
that log: 24 failures in r17-r19 replaced or moved it (all ten r17 passes
were false, the line came from the app
start). The request now says to keep the received log and add the filter
after it.
- transform-data-mapping seeded the finished shipment-mapping.groovy, so
the file check of step 2 passed before the
model acted and 14 of 18 failures in r17-r19 reused it instead of writing
it. It is now in exclude_seeds.
- The checkpoint after a failed step restored the route but not the data
files its route consumed (8 of 12 enrich
runs began step 3 without customers/*.json). It now copies back the seed
files that are gone.
Reference pass of the two examples: 7 of 7.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 31 +++++++++++++++++++++++++++++--
ai-benchmark/gen_stepwise.py | 6 ++++--
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py
b/ai-benchmark/agent_mcp_stepwise.py
index fee436a..4beeda6 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -13,7 +13,7 @@ Env: BENCH_MODEL (ollama model), MCP_URL (default
http://127.0.0.1:9090/mcp), BE
BENCH_STEPS (the steps file, default steps.json),
BENCH_TOOL_CALLS (max tool calls per step, default 25; 20 until Oct 15 2026).
"""
-import difflib, json, os, re, subprocess, sys, time, urllib.error,
urllib.request
+import difflib, json, os, re, shutil, subprocess, sys, time, urllib.error,
urllib.request
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from mcp_client import McpClient # noqa: E402
@@ -237,6 +237,29 @@ def error_snapshot(mcp, name):
return records, count, reloads, before_msgs
+def restore_seeds(project, cfg):
+ """Copies back the files of the example's seed that are gone from the
project, such as customer files a failed
+ step's route consumed (moved into customers/.camel when it had no noop),
so the checkpoint leaves the next step the
+ data the example starts with. Answers whether any file was copied (r19: 8
of 12 enrich runs began step 3 without
+ them)."""
+ seed = cfg.get("seed")
+ src = os.path.join(HERE, "seed", seed) if seed else None
+ if not src or not os.path.isdir(src):
+ return False
+ restored = False
+ for root, dirs, files in os.walk(src):
+ for f in files:
+ rel = os.path.relpath(os.path.join(root, f), src)
+ if rel.split(os.sep)[0] in cfg.get("exclude_seeds", []):
+ continue
+ dst = os.path.join(project, rel)
+ if not os.path.exists(dst):
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ shutil.copy2(os.path.join(root, f), dst)
+ restored = True
+ return restored
+
+
def checkpoint_restart(mcp, name, sid, log):
"""Restarts the app after the checkpoint of a restart step, and answers
its name (as the step restart does)."""
data, raw = jcall(mcp, "camel_control", {"name": name, "action":
"restart"})
@@ -691,9 +714,13 @@ def main():
# Without it the next step starts broken (r18: jms steps 2-4
without a connection factory,
# rest-code-first step 3 failing on the deleted src/main/java
Order.java)
restart_after = bool(cfg.get("restart_each_step") or
step.get("restart")) or removed
+ # after a failed step, the data files of the seed that its
route consumed come back too
+ restored = not res["ok"] and restore_seeds(project, cfg)
+ if restored:
+ print(f"step{sid}: checkpoint restored seed files",
file=log, flush=True)
if FAST:
# files that already equal the reference are not written
again: no reload, nothing to wait for
- if removed or any(read(os.path.join(project, f)) != c for
f, c in step["reference"].items()):
+ if removed or restored or any(read(os.path.join(project,
f)) != c for f, c in step["reference"].items()):
seen = error_snapshot(mcp, name)[2]
apply_reference(project, cfg, step["reference"], mcp,
name)
if restart_after:
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index 362ec29..6ac4ebe 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -293,7 +293,7 @@ EXAMPLES.append({
"seed": "route-filter-and-multicast",
"initial": {FM: route("filter-and-multicast", FILE_ORDERS, UNMARSHAL +
FM_LOG), "application.properties": ""},
"steps": [
- {"request": "Add a filter that only lets paid orders through (status
== 'paid') and log inside it \"Order <id> is paid: forwarding to the
departments\"; the pending order must not appear in that inner log.",
+ {"request": "Keep the received log, and after it add a filter that
only lets paid orders through (status == 'paid') and log inside it \"Order <id>
is paid: forwarding to the departments\"; the pending order must not appear in
that inner log.",
"check": {"file_regex": "filter", "log_regex": ["ORD-1001 is paid",
"ORD-1003 received, status pending"], "log_not_regex": "ORD-1003 is paid"},
"reference": {FM: fm_s2}},
{"request": "Add two routes, direct:warehouse (logs \"Warehouse: pick
<n> line(s) for <id>\") and direct:invoicing (logs \"Invoicing: bill customer
<customer> for <id>\"), and inside the filter replace the log with a multicast
to both.",
@@ -439,7 +439,9 @@ dm_s3 = route("data-mapping", TIMER_ONCE, BODY_ORDER +
UNMARSHAL + GROOVY_FILE +
dm_s5 = route("data-mapping", TIMER_ONCE, BODY_ORDER + UNMARSHAL + GROOVY_FILE
+ MARSHAL_JSON + log("Shipment for the courier: ${body}"))
EXAMPLES.append({
"name": "transform-data-mapping", "route_file": DM, "props_file":
"application.properties", "wait_seconds": 8,
- "seed": "transform-data-mapping",
+ # the seed has the finished script: step 2 asks for it to be written, and
its file check would pass before the
+ # model acts (r17-r19: 14 of 18 failures reused it instead)
+ "seed": "transform-data-mapping", "exclude_seeds":
["shipment-mapping.groovy"],
"initial": {DM: dm_s1, "application.properties": ""},
"steps": [
{"request": "After the unmarshal, set the body with an inline groovy
expression that returns the map [shipmentRef: body.orderId], and log
\"Shipment: ${body}\".",
