This is an automated email from the ASF dual-hosted git repository. rombert pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/sling-whiteboard.git
commit 3922cd0362d6542e7c55f37d7a5d1a1b11b2c814 Author: Robert Munteanu <[email protected]> AuthorDate: Wed Apr 22 14:46:03 2026 +0200 feat(skill_evals): make skills enabled a parameter --- skill-evals/README.md | 20 ++++++++++++++++++-- .../src/skill_evals/jcr_js_nodetypes/dataset.jsonl | 3 +-- skill-evals/src/skill_evals/jcr_js_nodetypes/task.py | 19 +++++++++++++------ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/skill-evals/README.md b/skill-evals/README.md index 402edf1c..610495ac 100644 --- a/skill-evals/README.md +++ b/skill-evals/README.md @@ -40,10 +40,26 @@ export INSPECT_EVAL_MODEL=bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 uv run inspect eval skill_evals/jcr_js_nodetypes ``` +By default the task runs without skills because `skill_enabled` defaults to `false`. + +Enable or disable skills explicitly with a task argument: + +```bash +uv run inspect eval skill_evals/jcr_js_nodetypes -T skill_enabled=true +uv run inspect eval skill_evals/jcr_js_nodetypes -T skill_enabled=false +``` + +You can also set the task argument through Inspect's standard environment variable: + +```bash +export INSPECT_EVAL_TASK_ARGS=skill_enabled=true +uv run inspect eval skill_evals/jcr_js_nodetypes +``` + You can also specify the desired model from the command line: ```bash -uv run inspect eval sling_evals/jcr_js_nodetypes \ +uv run inspect eval skill_evals/jcr_js_nodetypes \ --model my-model ``` @@ -58,7 +74,7 @@ Current status: - The sandbox image includes Maven 3, Java 8, the target repository, and a `mvn8` wrapper. - The sample setup script checks out git revision `52cea70b190c8389495a4fa494161eac13416891`, where the parent POM version is `22`. -- The prompt used for both samples is `update to the next parent pom version`. +- The prompt used for the sample is `update to the next parent pom version`. - A run is considered successful only if: - the parent POM version is updated to `23` - `mvn clean verify` succeeds diff --git a/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl b/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl index 57335b50..322c5fad 100644 --- a/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl +++ b/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl @@ -1,2 +1 @@ -{"id":"upgrade-parent-pom-no-skill","input":"update to the next parent pom version","skill_installed":false,"git_revision":"52cea70b190c8389495a4fa494161eac13416891","expected_parent_version":"23"} -{"id":"upgrade-parent-pom-with-skill","input":"update to the next parent pom version","skill_installed":true,"git_revision":"52cea70b190c8389495a4fa494161eac13416891","expected_parent_version":"23"} +{"id":"upgrade-parent-pom","input":"update to the next parent pom version","git_revision":"52cea70b190c8389495a4fa494161eac13416891","expected_parent_version":"23"} diff --git a/skill-evals/src/skill_evals/jcr_js_nodetypes/task.py b/skill-evals/src/skill_evals/jcr_js_nodetypes/task.py index e3912747..c3d22b63 100644 --- a/skill-evals/src/skill_evals/jcr_js_nodetypes/task.py +++ b/skill-evals/src/skill_evals/jcr_js_nodetypes/task.py @@ -22,13 +22,12 @@ def _skill_paths() -> list[Path]: return sorted(path for path in skills_dir.iterdir() if (path / "SKILL.md").is_file()) -def _record_to_sample(record: dict) -> Sample: - skill_installed = bool(record["skill_installed"]) +def _record_to_sample(record: dict, *, skill_enabled: bool) -> Sample: return Sample( id=record.get("id"), input=record["input"], metadata={ - "skill_installed": skill_installed, + "skill_enabled": skill_enabled, "git_revision": record["git_revision"], "expected_parent_version": record["expected_parent_version"], }, @@ -38,10 +37,16 @@ def _record_to_sample(record: dict) -> Sample: @task -def jcr_js_nodetypes() -> Task: - dataset = json_dataset(str(_task_dir() / "dataset.jsonl"), sample_fields=_record_to_sample) +def jcr_js_nodetypes(skill_enabled: bool = False) -> Task: + dataset = json_dataset( + str(_task_dir() / "dataset.jsonl"), + sample_fields=lambda record: _record_to_sample( + record, skill_enabled=skill_enabled + ), + ) skill_paths = _skill_paths() + task_name_suffix = "skills" if skill_enabled else "no skills" @solver def sample_solver(): @@ -50,7 +55,7 @@ def jcr_js_nodetypes() -> Task: bash(timeout=600), text_editor(timeout=600), ] - if state.metadata and state.metadata.get("skill_installed") and skill_paths: + if state.metadata and state.metadata.get("skill_enabled") and skill_paths: tools.append(skill(skill_paths, dir="/workspace/skills")) agent = react( @@ -66,10 +71,12 @@ def jcr_js_nodetypes() -> Task: return solve return Task( + name=f"jcr_js_nodetypes [{task_name_suffix}]", dataset=dataset, solver=sample_solver(), scorer=parent_pom_update(), sandbox=("docker", str(_task_dir() / "compose.yaml")), + metadata={"skill_enabled": skill_enabled}, epochs=5, time_limit=600, )
