This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit a16e65ac746092c0a6087eb8e67212314d356073 Author: Claus Ibsen <[email protected]> AuthorDate: Sun Oct 18 15:33:21 2026 +0200 ai-benchmark: ai/ticket-classifier in the new set, and files checks accept a glob ticket-classifier: a local model (granite4:3b on the host Ollama, next to the agent's model) names the category of each ticket through langchain4j-chat, the ticket is filed in tickets-out/<category>/, and a circuit breaker with a timeout files it under unknown. The model writes its own prompt, so the checks accept any of the three categories and the filed ticket in any category folder: a files check name may be a glob (tickets-out/*/T-1.json). The final reference is the example's route file (apache/camel-jbang-examples#119). Reference pass 3/3 twice. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m --- ai-benchmark/agent_mcp_stepwise.py | 23 ++++++---- ai-benchmark/gen_stepwise.py | 53 ++++++++++++++++++++++ .../seed/ai-ticket-classifier/tickets/T-1.json | 1 + .../seed/ai-ticket-classifier/tickets/T-2.json | 1 + .../seed/ai-ticket-classifier/tickets/T-3.json | 1 + 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index c4875e1..9ae4abb 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -237,6 +237,15 @@ def error_snapshot(mcp, name): return records, count, reloads, before_msgs +def file_matches(project, fname, rx): + """Whether a file of a check holds the regex. A name with * is a glob in the project, and any match counts (the + category folder of ticket-classifier, which the model's answer picks, Oct 18); else project_file.""" + if "*" in fname: + import glob + return any(re.search(rx, read(f) or "") for f in glob.glob(os.path.join(project, fname))) + return bool(re.search(rx, project_file(project, fname) or "")) + + def project_file(project, fname): """A file of the project by path, else by name anywhere in it (as score reads the files of a check).""" content = read(os.path.join(project, fname)) @@ -277,7 +286,7 @@ def wait_for_check(mcp, name, chk, cfg, project, before, reload_at): while time.time() - t0 < limit: fresh = [msg(l) for l in log_lines(mcp, name, LOG_WINDOW) if fresh_ok(l)] logs_ok = all(sum(1 for m in fresh if re.search(rx, m, re.S)) >= need for rx in regexes) - files_ok = all(re.search(rx, project_file(project, f) or "") for f, rx in files.items()) + files_ok = all(file_matches(project, f, rx) for f, rx in files.items()) if logs_ok and files_ok: time.sleep(max(0.0, min(FAST_GRACE, limit - (time.time() - t0)))) return round(time.time() - t0, 1) @@ -381,15 +390,9 @@ def score(step, project, cfg, mcp, name, before): # (after the wait: a file the app writes on the reload, an outbox or an archive, exists only then) # round 2: other files of the project (a Java bean, beans.yaml, a stylesheet, a dropped data file) for fname, rx in (chk.get("files") or {}).items(): - # by path, else by name anywhere in the project (a class the model put under src/main/java is the same class) - content = read(os.path.join(project, fname)) - if not content: - for root, dirs, files in os.walk(project): - dirs[:] = [d for d in dirs if not d.startswith(".")] - if os.path.basename(fname) in files: - content = read(os.path.join(root, os.path.basename(fname))) - break - if not re.search(rx, content): + # by path, else by name anywhere in the project (a class the model put under src/main/java is the same class); + # a name with * is a glob + if not file_matches(project, fname, rx): result["file_ok"] = False lines = log_lines(mcp, name, LOG_WINDOW) def lvl(l): return (l.get("level") or "").upper() diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py index 93f3cd6..dcc1fce 100644 --- a/ai-benchmark/gen_stepwise.py +++ b/ai-benchmark/gen_stepwise.py @@ -1043,6 +1043,59 @@ EXAMPLES.append({ ]}) +# ---------------------------------------------------------------- ai/ticket-classifier (a local model in a route) +# The route's model is granite4:3b on the host Ollama, next to the agent's model (64 GB hold both). The model writes +# its own prompt, so the category of a ticket can differ: the checks accept any of the three, and the filed ticket in +# any category folder (a glob in files). +TC = "ticket-classifier.camel.yaml" +TC_PROPS = "ollama.base.url=http://localhost:11434\nollama.model=granite4:3b\n"; + DATA_DIR + "tickets-out\n" +TC_BEANS = ("- beans:\n - name: chatModel\n type: dev.langchain4j.model.ollama.OllamaChatModel\n" + " builderClass: dev.langchain4j.model.ollama.OllamaChatModel$OllamaChatModelBuilder\n builderMethod: build\n" + " properties:\n baseUrl: \"{{ollama.base.url}}\"\n modelName: \"{{ollama.model}}\"\n temperature: 0.0\n\n") +FILE_TICKETS = " uri: file\n parameters:\n directoryName: tickets\n noop: true\n sortBy: \"file:name\"\n" +def tc_ask(ind): + i = " " * ind + return (i + "- setVariable:\n" + i + " name: prompt\n" + i + " expression:\n" + i + " simple:\n" + i + " expression: >-\n" + + i + " You sort the support tickets of a web shop. Answer with one word only, the category of the\n" + + i + " ticket: shipping, billing or account. Ticket: ${body[text]}\n" + + i + "- to:\n" + i + " uri: langchain4j-chat\n" + i + " parameters:\n" + i + " chatId: tickets\n" + + i + " chatModel: \"#chatModel\"\n" + i + " variableSend: prompt\n" + i + " variableReceive: answer\n" + + i + "- setVariable:\n" + i + " name: category\n" + i + " expression:\n" + i + " simple:\n" + + i + " expression: \"${variable.answer.toLowerCase().replaceAll('[^a-z]', '')}\"\n") +TC_BREAKER = (" - circuitBreaker:\n resilience4jConfiguration:\n timeoutEnabled: true\n timeoutDuration: 60000\n" + " steps:\n" + tc_ask(14) + " onFallback:\n steps:\n - setVariable:\n" + " name: category\n expression:\n constant:\n expression: unknown\n") +TC_UNKNOWN = (" - filter:\n expression:\n simple:\n expression: \"${variable.category} not in 'shipping,billing,account'\"\n" + " steps:\n - setVariable:\n name: category\n expression:\n constant:\n" + " expression: unknown\n") +TC_FILE = (set_header("CamelFileName", "simple", "\"${variable.category}/${body[ticketId]}.json\"") + + " - marshal:\n json:\n library: Jackson\n" + + " - to:\n uri: file\n parameters:\n directoryName: tickets-out\n") +TC_LOG = log("Ticket ${body[ticketId]} is ${variable.category}") +TC_RECEIVED = UNMARSHAL + log("Ticket ${body[ticketId]}: ${body[text]}") +tc_s1 = route("classify-tickets", FILE_TICKETS, TC_RECEIVED) +tc_s2 = TC_BEANS + route("classify-tickets", FILE_TICKETS, TC_RECEIVED + tc_ask(8) + TC_LOG) +tc_s3 = TC_BEANS + route("classify-tickets", FILE_TICKETS, TC_RECEIVED + tc_ask(8) + TC_UNKNOWN + TC_LOG + TC_FILE) +tc_s4 = TC_BEANS + route("classify-tickets", FILE_TICKETS, TC_RECEIVED + TC_BREAKER + TC_UNKNOWN + TC_LOG + TC_FILE) +TC_CATEGORY = "(?i)Ticket %s is (shipping|billing|account)\\b" +EXAMPLES.append({ + "name": "ai-ticket-classifier", "route_file": TC, "props_file": "application.properties", "wait_seconds": 20, + "seed": "ai-ticket-classifier", "steps_dir": NEW, + "initial": {TC: tc_s1, "application.properties": TC_PROPS}, + "steps": [ + {"request": "Ask the local model for the category of each ticket: shipping, billing or account. Use the langchain4j-chat component with an Ollama chat model (dev.langchain4j.model.ollama.OllamaChatModel) for the model and base URL in application.properties. Keep the ticket as the body, the category in a variable category, and log \"Ticket ${body[ticketId]} is ${variable.category}\".", + "check": {"file_regex": "langchain4j-chat", "log_regex": [TC_CATEGORY % "T-1", TC_CATEGORY % "T-3"]}, + "reference": {TC: tc_s2}}, + {"request": "File each ticket as JSON in tickets-out/<category>/<ticketId>.json, and in tickets-out/unknown/ when the category is not shipping, billing or account.", + "check": {"files": {"tickets-out/*/T-1.json": "T-1", "tickets-out/*/T-3.json": "T-3"}, "log_regex": TC_CATEGORY % "T-2"}, + "reference": {TC: tc_s3}}, + {"request": "Put the call to the model in a circuit breaker with a timeout of 60 seconds; on fallback, the category is unknown.", + "check": {"file_regex": "circuitBreaker", "file_regex2": "onFallback", "files": {"tickets-out/*/T-1.json": "T-1"}, + "log_regex": TC_CATEGORY % "T-1"}, + "reference": {TC: tc_s4}}, + ]}) + + def write_project(ex): d = os.path.join(PROJECTS, ex["name"]) shutil.rmtree(d, ignore_errors=True) diff --git a/ai-benchmark/seed/ai-ticket-classifier/tickets/T-1.json b/ai-benchmark/seed/ai-ticket-classifier/tickets/T-1.json new file mode 100644 index 0000000..48609de --- /dev/null +++ b/ai-benchmark/seed/ai-ticket-classifier/tickets/T-1.json @@ -0,0 +1 @@ +{"ticketId": "T-1", "customer": "C-482", "text": "My order ORD-1001 has not arrived yet. It is two weeks now, where is my parcel?"} diff --git a/ai-benchmark/seed/ai-ticket-classifier/tickets/T-2.json b/ai-benchmark/seed/ai-ticket-classifier/tickets/T-2.json new file mode 100644 index 0000000..d741186 --- /dev/null +++ b/ai-benchmark/seed/ai-ticket-classifier/tickets/T-2.json @@ -0,0 +1 @@ +{"ticketId": "T-2", "customer": "C-207", "text": "I was charged twice for order ORD-1002, please refund one payment."} diff --git a/ai-benchmark/seed/ai-ticket-classifier/tickets/T-3.json b/ai-benchmark/seed/ai-ticket-classifier/tickets/T-3.json new file mode 100644 index 0000000..0a6f313 --- /dev/null +++ b/ai-benchmark/seed/ai-ticket-classifier/tickets/T-3.json @@ -0,0 +1 @@ +{"ticketId": "T-3", "customer": "C-134", "text": "How do I change the email address and the password of my account?"}
