This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
from 0570fd9 README: point at the ai-benchmark harness
new 8c2ddea ai-benchmark: the tool allow-list names the shared authoring
tools only (CAMEL-24712 removes the older per-kind ones)
new eae5b3e The stepwise ladder: the round-2 harness that scores one
example step by step
new 1b947aa Round 2 of the one-shot suite: k runs, pass@k and pass^k, a
held-out set and services
new 39312f2 The ladder gives the model 20 tool calls per step, not 12
new a9de890 The sql rung runs on Postgres with camel infra, as the
example ships it
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ai-benchmark/.gitignore | 7 +
ai-benchmark/README.md | 24 +-
ai-benchmark/agent_local.py | 253 ++++++-
ai-benchmark/agent_mcp_stepwise.py | 348 ++++++++-
ai-benchmark/examples-intermediate.json | 61 ++
ai-benchmark/examples-ladder-10.json | 235 ++++++
ai-benchmark/examples-ladder-dry.json | 109 +++
ai-benchmark/examples-ladder.json | 508 +++++++++++++
ai-benchmark/gen_ladder.py | 132 ++++
ai-benchmark/gen_local.py | 7 +-
ai-benchmark/gen_stepwise.py | 825 +++++++++++++++++++++
ai-benchmark/passk.py | 38 +
ai-benchmark/run-stepwise-ladder.sh | 35 +
ai-benchmark/run-suite.sh | 43 +-
ai-benchmark/run_one.sh | 8 +-
.../_peer-openapi-server/application.properties | 1 +
.../_peer-openapi-server/openapi-server.camel.yaml | 162 ++++
.../seed/_peer-openapi-server/stock-api.json | 201 +++++
ai-benchmark/seed/_peer-openapi-server/stock.json | 5 +
.../samples/invoice-2001.json | 1 +
.../samples/invoice-2002.json | 1 +
.../samples/invoice-2003.json | 1 +
.../samples/invoice-2004.json | 1 +
.../connect-file-processing/samples/note-2005.txt | 1 +
.../connect-http-client/orders/order-1001.json | 2 +
.../connect-http-client/orders/order-1002.json | 2 +
.../connect-http-client/orders/order-1003.json | 2 +
ai-benchmark/seed/connect-http-client/stock.json | 5 +
.../connect-service-sql/orders/order-1001.json | 2 +
.../connect-service-sql/orders/order-1002.json | 2 +
.../connect-service-sql/orders/order-1003.json | 2 +
ai-benchmark/seed/connect-stock-api/stock.json | 5 +
.../orders/order-1001.json | 2 +
.../orders/order-1002.json | 2 +
.../orders/order-1003.json | 2 +
.../seed/contracts-openapi-client/stock-api.json | 201 +++++
.../seed/contracts-openapi-server/stock-api.json | 201 +++++
.../seed/contracts-openapi-server/stock.json | 5 +
.../orders/order-1001.json | 2 +
.../orders/order-1002.json | 2 +
.../orders/order-1003.json | 2 +
.../seed/openapi-client/examples/1001.json | 18 +
ai-benchmark/seed/openapi-client/petstore-api.json | 299 ++++++++
.../.camel-jbang/camel-jbang-run.properties | 40 +
.../seed/openapi-server-ref/application.properties | 18 +
.../openapi-server-ref/openapi-server.camel.yaml | 162 ++++
.../seed/openapi-server-ref/stock-api.json | 201 +++++
ai-benchmark/seed/openapi-server-ref/stock.json | 5 +
.../seed/openapi-server/examples/pet/1000.json | 18 +
ai-benchmark/seed/openapi-server/petstore-api.json | 299 ++++++++
.../seed/route-aggregator/orders/order-1001.json | 2 +
.../seed/route-aggregator/orders/order-1002.json | 2 +
.../seed/route-aggregator/orders/order-1003.json | 2 +
.../orders/order-1001.json | 2 +
.../orders/order-1002.json | 2 +
.../orders/order-1003.json | 2 +
.../orders/order-1001.json | 2 +
.../orders/order-1002.json | 2 +
.../orders/order-1003.json | 2 +
.../seed/route-order-lines/orders/order-1001.json | 2 +
.../seed/route-order-lines/orders/order-1002.json | 2 +
.../seed/route-order-lines/orders/order-1003.json | 2 +
.../seed/transform-csv-to-json/inbox/invoices.csv | 4 +
.../seed/transform-data-mapping/order.json | 3 +
.../transform-data-mapping/shipment-mapping.groovy | 12 +
.../seed/transform-groovy/order-bad-email.json | 2 +
ai-benchmark/seed/transform-groovy/order.json | 2 +
.../seed/transform-json-transform/order.json | 3 +
.../transform-xml-to-json/inbox/supplier-order.xml | 7 +
.../seed/transform-xslt/inbox/supplier-order.xml | 7 +
ai-benchmark/seed/transform-xslt/packing-slip.xsl | 15 +
ai-benchmark/steps.json | 7 +-
ai-benchmark/summarize_stepwise.py | 25 +
73 files changed, 4544 insertions(+), 75 deletions(-)
create mode 100644 ai-benchmark/examples-intermediate.json
create mode 100644 ai-benchmark/examples-ladder-10.json
create mode 100644 ai-benchmark/examples-ladder-dry.json
create mode 100644 ai-benchmark/examples-ladder.json
create mode 100755 ai-benchmark/gen_ladder.py
create mode 100644 ai-benchmark/gen_stepwise.py
create mode 100755 ai-benchmark/passk.py
create mode 100755 ai-benchmark/run-stepwise-ladder.sh
create mode 100644
ai-benchmark/seed/_peer-openapi-server/application.properties
create mode 100644
ai-benchmark/seed/_peer-openapi-server/openapi-server.camel.yaml
create mode 100644 ai-benchmark/seed/_peer-openapi-server/stock-api.json
create mode 100644 ai-benchmark/seed/_peer-openapi-server/stock.json
create mode 100644
ai-benchmark/seed/connect-file-processing/samples/invoice-2001.json
create mode 100644
ai-benchmark/seed/connect-file-processing/samples/invoice-2002.json
create mode 100644
ai-benchmark/seed/connect-file-processing/samples/invoice-2003.json
create mode 100644
ai-benchmark/seed/connect-file-processing/samples/invoice-2004.json
create mode 100644
ai-benchmark/seed/connect-file-processing/samples/note-2005.txt
create mode 100644 ai-benchmark/seed/connect-http-client/orders/order-1001.json
create mode 100644 ai-benchmark/seed/connect-http-client/orders/order-1002.json
create mode 100644 ai-benchmark/seed/connect-http-client/orders/order-1003.json
create mode 100644 ai-benchmark/seed/connect-http-client/stock.json
create mode 100644 ai-benchmark/seed/connect-service-sql/orders/order-1001.json
create mode 100644 ai-benchmark/seed/connect-service-sql/orders/order-1002.json
create mode 100644 ai-benchmark/seed/connect-service-sql/orders/order-1003.json
create mode 100644 ai-benchmark/seed/connect-stock-api/stock.json
create mode 100644
ai-benchmark/seed/contracts-openapi-client/orders/order-1001.json
create mode 100644
ai-benchmark/seed/contracts-openapi-client/orders/order-1002.json
create mode 100644
ai-benchmark/seed/contracts-openapi-client/orders/order-1003.json
create mode 100644 ai-benchmark/seed/contracts-openapi-client/stock-api.json
create mode 100644 ai-benchmark/seed/contracts-openapi-server/stock-api.json
create mode 100644 ai-benchmark/seed/contracts-openapi-server/stock.json
create mode 100644
ai-benchmark/seed/fail-well-error-handling/orders/order-1001.json
create mode 100644
ai-benchmark/seed/fail-well-error-handling/orders/order-1002.json
create mode 100644
ai-benchmark/seed/fail-well-error-handling/orders/order-1003.json
create mode 100644 ai-benchmark/seed/openapi-client/examples/1001.json
create mode 100644 ai-benchmark/seed/openapi-client/petstore-api.json
create mode 100644
ai-benchmark/seed/openapi-server-ref/.camel-jbang/camel-jbang-run.properties
create mode 100644 ai-benchmark/seed/openapi-server-ref/application.properties
create mode 100644
ai-benchmark/seed/openapi-server-ref/openapi-server.camel.yaml
create mode 100644 ai-benchmark/seed/openapi-server-ref/stock-api.json
create mode 100644 ai-benchmark/seed/openapi-server-ref/stock.json
create mode 100644 ai-benchmark/seed/openapi-server/examples/pet/1000.json
create mode 100644 ai-benchmark/seed/openapi-server/petstore-api.json
create mode 100644 ai-benchmark/seed/route-aggregator/orders/order-1001.json
create mode 100644 ai-benchmark/seed/route-aggregator/orders/order-1002.json
create mode 100644 ai-benchmark/seed/route-aggregator/orders/order-1003.json
create mode 100644
ai-benchmark/seed/route-content-based-router/orders/order-1001.json
create mode 100644
ai-benchmark/seed/route-content-based-router/orders/order-1002.json
create mode 100644
ai-benchmark/seed/route-content-based-router/orders/order-1003.json
create mode 100644
ai-benchmark/seed/route-filter-and-multicast/orders/order-1001.json
create mode 100644
ai-benchmark/seed/route-filter-and-multicast/orders/order-1002.json
create mode 100644
ai-benchmark/seed/route-filter-and-multicast/orders/order-1003.json
create mode 100644 ai-benchmark/seed/route-order-lines/orders/order-1001.json
create mode 100644 ai-benchmark/seed/route-order-lines/orders/order-1002.json
create mode 100644 ai-benchmark/seed/route-order-lines/orders/order-1003.json
create mode 100644 ai-benchmark/seed/transform-csv-to-json/inbox/invoices.csv
create mode 100644 ai-benchmark/seed/transform-data-mapping/order.json
create mode 100644
ai-benchmark/seed/transform-data-mapping/shipment-mapping.groovy
create mode 100644 ai-benchmark/seed/transform-groovy/order-bad-email.json
create mode 100644 ai-benchmark/seed/transform-groovy/order.json
create mode 100644 ai-benchmark/seed/transform-json-transform/order.json
create mode 100644
ai-benchmark/seed/transform-xml-to-json/inbox/supplier-order.xml
create mode 100644 ai-benchmark/seed/transform-xslt/inbox/supplier-order.xml
create mode 100644 ai-benchmark/seed/transform-xslt/packing-slip.xsl
create mode 100644 ai-benchmark/summarize_stepwise.py
