This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
The following commit(s) were added to refs/heads/main by this push:
new ede2cf9 ai-benchmark: restart after the checkpoint of a restart step,
and no kafka pollTimeoutMs workaround
ede2cf9 is described below
commit ede2cf96ade9dcd841bf1dc9ef00d39e9be2dfa5
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Oct 19 09:48:32 2026 +0200
ai-benchmark: restart after the checkpoint of a restart step, and no kafka
pollTimeoutMs workaround
After each step the harness writes the reference files, so the next step
starts right. For a restart step that was
only a reload, which does not apply new camel.beans properties
(CAMEL-25767) and keeps a Java file the harness
removed, so the next reload fails on it. In r18 the next step then started
broken: jms-request-reply steps 2 to 4
without a connection factory (about 99 errors before the model did
anything), and rest-code-first step 3 failing on
the deleted src/main/java Order.java. The checkpoint of a restart step, or
one that removed a file, now restarts the
app. Checked with one model run of jms-request-reply: step 1 failed, the
checkpoint restarted the app, step 2 started
with the orders on the queue, and step 4 passed.
kafka-orders needs no pollTimeoutMs=500 any more: since CAMEL-25753 a Kafka
consumer stops without waiting for its
current poll. Reference pass 3/3 without errors on main.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 45 ++++++++++++++++++++++++++++----------
ai-benchmark/gen_stepwise.py | 8 +++----
2 files changed, 36 insertions(+), 17 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py
b/ai-benchmark/agent_mcp_stepwise.py
index 9ae4abb..fee436a 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -237,6 +237,14 @@ def error_snapshot(mcp, name):
return records, count, reloads, before_msgs
+def checkpoint_restart(mcp, name, sid, log):
+ """Restarts the app after the checkpoint of a restart step, and answers
its name (as the step restart does)."""
+ data, raw = jcall(mcp, "camel_control", {"name": name, "action":
"restart"})
+ print(f"step{sid}: checkpoint restart -> {raw[:200]}", file=log,
flush=True)
+ time.sleep(8)
+ return data["name"] if isinstance(data, dict) and data.get("name") else
name
+
+
def file_matches(project, fname, rx):
"""Whether a file of a check holds the regex. A name with * is a glob in
the project, and any match counts (the
category folder of ticket-classifier, which the model's answer picks, Oct
18); else project_file."""
@@ -668,6 +676,7 @@ def main():
+ f"changed_lines={res['changed_lines']} calls={calls}
writes={writes} refused={refused} secs={res['seconds']} tokens={tokens}",
file=log, flush=True)
trace.close()
if step.get("reference"):
+ removed = False
for fname in step["reference"]:
# a same-named file elsewhere (the model's copy of a Java
class under src/main/java) would be a
# second class of the same name after the fix: remove it
@@ -676,25 +685,37 @@ def main():
other = os.path.join(root, os.path.basename(fname))
if os.path.basename(fname) in files and
os.path.abspath(other) != os.path.abspath(os.path.join(project, fname)):
os.remove(other)
+ removed = True
+ # the checkpoint of a restart step needs the restart too: a
reload does not apply new camel.beans
+ # properties (CAMEL-25767) and keeps a Java file the harness
removed, so the next reload fails on it.
+ # Without it the next step starts broken (r18: jms steps 2-4
without a connection factory,
+ # rest-code-first step 3 failing on the deleted src/main/java
Order.java)
+ restart_after = bool(cfg.get("restart_each_step") or
step.get("restart")) or removed
if FAST:
# files that already equal the reference are not written
again: no reload, nothing to wait for
- if any(read(os.path.join(project, f)) != c for f, c in
step["reference"].items()):
+ if removed or any(read(os.path.join(project, f)) != c for
f, c in step["reference"].items()):
seen = error_snapshot(mcp, name)[2]
apply_reference(project, cfg, step["reference"], mcp,
name)
- for _ in range(int(24 / FAST_POLL)):
- time.sleep(FAST_POLL)
- if any(reload_key(l) not in seen for l in
log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
- break
+ if restart_after:
+ name = checkpoint_restart(mcp, name, sid, log)
+ else:
+ for _ in range(int(24 / FAST_POLL)):
+ time.sleep(FAST_POLL)
+ if any(reload_key(l) not in seen for l in
log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
+ break
res["settled"] = wait_quiet(mcp, name, FAST_QUIET, 6)
else:
apply_reference(project, cfg, step["reference"], mcp, name)
- # the checkpoint write reloads the app too: wait for that
reload and let its consumers settle, so
- # the next step's error and log snapshot is taken in a
quiet state
- seen = error_snapshot(mcp, name)[2]
- for _ in range(12):
- time.sleep(2)
- if any(reload_key(l) not in seen for l in
log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
- break
+ if restart_after:
+ name = checkpoint_restart(mcp, name, sid, log)
+ else:
+ # the checkpoint write reloads the app too: wait for
that reload and let its consumers settle,
+ # so the next step's error and log snapshot is taken
in a quiet state
+ seen = error_snapshot(mcp, name)[2]
+ for _ in range(12):
+ time.sleep(2)
+ if any(reload_key(l) not in seen for l in
log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
+ break
time.sleep(6)
if not res["ok"]:
messages.append({"role": "user", "content": "I fixed that
step myself; the files now contain the correct version. Continue with the next
request."})
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index dcc1fce..362ec29 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -784,11 +784,9 @@ NEW = "steps-new"
# ----------------------------------------------------------------
connect-service/kafka-orders (Kafka started with camel infra)
KO = "kafka-orders.camel.yaml"
-# pollTimeoutMs 500: a camel-kafka consumer stops only after its current poll
returns (the poll loop holds the lock that
-# stop() waits for before it wakes the consumer), up to the default 5000 ms
per consumer and one route after another;
-# the 5 s shutdown timeout of camel run --dev is then exceeded on a reload
with two or more Kafka routes, and the
-# forced shutdown logs ERRORs from the interrupted Kafka clients (kref1, Oct
18). Remove once CAMEL-25753 is fixed.
-KO_PROPS =
"camel.component.kafka.brokers=localhost:9092\ncamel.component.kafka.pollTimeoutMs=500\n"
+# no pollTimeoutMs: since CAMEL-25753 (Oct 18) a camel-kafka consumer stops
without waiting for its current poll, so a
+# reload with several Kafka routes ends within the 5 s shutdown timeout of
camel run --dev
+KO_PROPS = "camel.component.kafka.brokers=localhost:9092\n"
def kafka_from(topic):
# earliest: every reload starts the consumers in a new random group, which
then reads the topic from the start
return " uri: kafka\n parameters:\n topic: " + topic +
"\n autoOffsetReset: earliest\n"
