From e89b428368369aa58ce468090e61f904f756c3b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sun, 14 Jun 2026 15:04:58 +0800 Subject: [PATCH 01/16] Add skill-eval harness scaffold with promptfoo --- dev/skill-evals/cases/command-routing.yaml | 57 ++++++++ dev/skill-evals/eval.sh | 144 +++++++++++++++++++++ dev/skill-evals/promptfooconfig.full.yaml | 89 +++++++++++++ dev/skill-evals/promptfooconfig.yaml | 67 ++++++++++ 4 files changed, 357 insertions(+) create mode 100644 dev/skill-evals/cases/command-routing.yaml create mode 100755 dev/skill-evals/eval.sh create mode 100644 dev/skill-evals/promptfooconfig.full.yaml create mode 100644 dev/skill-evals/promptfooconfig.yaml diff --git a/dev/skill-evals/cases/command-routing.yaml b/dev/skill-evals/cases/command-routing.yaml new file mode 100644 index 0000000000000..8c6ffb6121e90 --- /dev/null +++ b/dev/skill-evals/cases/command-routing.yaml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Command routing: does the agent choose uv / breeze / prek correctly? + +- description: "Core unit test: should use uv" + vars: + request: | + Run the tests in airflow-core/tests/models/test_taskinstance.py. + The file contains @pytest.mark.db_test markers on most test classes. + The project root is airflow-core. + You are on the host, not inside a Breeze container. + assert: + - type: javascript + value: 'output.runner === "uv"' + - type: javascript + value: 'output.command.includes("uv run --project airflow-core pytest")' + +- description: "Helm test: always breeze" + vars: + request: | + Run the Helm chart tests. + The change is in chart/templates/webserver/webserver-deployment.yaml. + Helm tests are in chart/tests/. + You are on the host, not inside a Breeze container. + assert: + - type: javascript + value: 'output.runner === "breeze"' + - type: javascript + value: 'output.command.includes("breeze testing helm-tests")' + +- description: "mypy type check: use prek" + vars: + request: | + Run mypy type checking on the airflow-core package. + The change is in airflow-core/src/airflow/models/dag.py. + You are on the host, not inside a Breeze container. + The project uses prek hooks for static checks. + assert: + - type: javascript + value: 'output.runner === "prek"' + - type: javascript + value: 'output.command.includes("prek run mypy-airflow-core")' diff --git a/dev/skill-evals/eval.sh b/dev/skill-evals/eval.sh new file mode 100755 index 0000000000000..4120f84fc7e48 --- /dev/null +++ b/dev/skill-evals/eval.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Airflow skill-eval harness +# +# Usage: +# ./eval.sh Quick: 2 arms (before vs after your change) +# ./eval.sh --full Full: 4 arms (+ baseline, skill-only) +# ./eval.sh --no-cache Disable promptfoo cache +# ./eval.sh --repeat 3 Run each case N times +# +# Auto-detects changes to AGENTS.md and SKILL.md against main branch. +# "before" = main's version, "after" = working tree. +# +# Requires: ANTHROPIC_API_KEY, npx, git + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SKILL_SRC="$REPO_ROOT/files/agent-skill-dev/airflow-contribution/SKILL.md" +AGENTS_SRC="$REPO_ROOT/AGENTS.md" +PROMPTFOO_VERSION="0.120.19" + +# Parse --full flag, pass everything else to promptfoo +FULL_MODE=false +PROMPTFOO_ARGS=() +for arg in "$@"; do + if [ "$arg" = "--full" ]; then + FULL_MODE=true + else + PROMPTFOO_ARGS+=("$arg") + fi +done + +# Resolve base branch +BASE_BRANCH="main" +if ! git -C "$REPO_ROOT" rev-parse --verify "$BASE_BRANCH" &>/dev/null; then + BASE_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)" + echo "Warning: 'main' not found, using current branch '$BASE_BRANCH' as base" +fi + +# Create temp directory, clean up on exit +ARMS_DIR="$(mktemp -d)" +trap 'rm -rf "$ARMS_DIR"' EXIT + +# Helper: place skill in a working dir +place_skill() { + local dir="$1" + local source="$2" # path to SKILL.md + mkdir -p "$dir/.claude/skills/airflow-contribution" + cp "$source" "$dir/.claude/skills/airflow-contribution/SKILL.md" +} + +# Helper: place AGENTS.md in a working dir +place_agents() { + local dir="$1" + local source="$2" # path to AGENTS.md + cp "$source" "$dir/AGENTS.md" +} + +# Get "before" versions from git +BEFORE_DIR="$(mktemp -d)" +trap 'rm -rf "$ARMS_DIR" "$BEFORE_DIR"' EXIT + +git -C "$REPO_ROOT" show "$BASE_BRANCH:AGENTS.md" > "$BEFORE_DIR/AGENTS.md" 2>/dev/null || \ + cp "$AGENTS_SRC" "$BEFORE_DIR/AGENTS.md" + +git -C "$REPO_ROOT" show "$BASE_BRANCH:files/agent-skill-dev/airflow-contribution/SKILL.md" \ + > "$BEFORE_DIR/SKILL.md" 2>/dev/null || \ + cp "$SKILL_SRC" "$BEFORE_DIR/SKILL.md" + +echo "Assembling arms ..." + +# === Quick mode arms (always built) === + +# before: main's AGENTS.md + main's skill +mkdir -p "$ARMS_DIR/before" +place_skill "$ARMS_DIR/before" "$BEFORE_DIR/SKILL.md" +place_agents "$ARMS_DIR/before" "$BEFORE_DIR/AGENTS.md" + +# after: working tree's AGENTS.md + working tree's skill +mkdir -p "$ARMS_DIR/after" +place_skill "$ARMS_DIR/after" "$SKILL_SRC" +place_agents "$ARMS_DIR/after" "$AGENTS_SRC" + +# === Full mode arms (only with --full) === +if [ "$FULL_MODE" = true ]; then + # baseline: no skill, no AGENTS.md + mkdir -p "$ARMS_DIR/baseline" + + # skill-only: working tree's skill, no AGENTS.md + mkdir -p "$ARMS_DIR/skill-only" + place_skill "$ARMS_DIR/skill-only" "$SKILL_SRC" + + # agents-only: working tree's AGENTS.md, no skill + mkdir -p "$ARMS_DIR/agents-only" + place_agents "$ARMS_DIR/agents-only" "$AGENTS_SRC" + + CONFIG="$SCRIPT_DIR/promptfooconfig.full.yaml" + echo "Mode: full (5 arms)" +else + CONFIG="$SCRIPT_DIR/promptfooconfig.yaml" + echo "Mode: quick (2 arms: before vs after)" +fi + +# Show what changed +echo "" +echo "Changes detected (vs $BASE_BRANCH):" +if ! diff -q "$BEFORE_DIR/AGENTS.md" "$AGENTS_SRC" &>/dev/null; then + echo " AGENTS.md — modified" +else + echo " AGENTS.md — unchanged" +fi +if ! diff -q "$BEFORE_DIR/SKILL.md" "$SKILL_SRC" &>/dev/null; then + echo " SKILL.md — modified" +else + echo " SKILL.md — unchanged" +fi +echo "" + +# Run +ARMS_DIR="$ARMS_DIR" \ + npx "promptfoo@$PROMPTFOO_VERSION" eval \ + -c "$CONFIG" \ + "${PROMPTFOO_ARGS[@]}" + +echo "" +echo "View results: npx promptfoo@$PROMPTFOO_VERSION view" diff --git a/dev/skill-evals/promptfooconfig.full.yaml b/dev/skill-evals/promptfooconfig.full.yaml new file mode 100644 index 0000000000000..bac0861808af3 --- /dev/null +++ b/dev/skill-evals/promptfooconfig.full.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Full mode: 5-arm research matrix +# +# before = main's AGENTS.md + main's SKILL.md +# after = working tree's AGENTS.md + working tree's SKILL.md +# baseline = no guidance at all +# skill-only = skill without AGENTS.md +# agents-only = AGENTS.md without skill +# +# Do not run directly — use eval.sh --full + +x-output-schema: &outputSchema + type: json_schema + schema: + type: object + required: [runner, command, rationale] + additionalProperties: false + properties: + runner: + type: string + enum: [uv, breeze, prek] + command: + type: string + rationale: + type: string + +x-base: &base + model: claude-sonnet-4-6 + setting_sources: ['project'] + append_allowed_tools: ['Read', 'Grep', 'Glob'] + output_format: *outputSchema + +providers: + # Core comparison + - id: anthropic:claude-agent-sdk + label: before + config: + <<: *base + working_dir: ${ARMS_DIR}/before + skills: ['airflow-contribution'] + + - id: anthropic:claude-agent-sdk + label: after + config: + <<: *base + working_dir: ${ARMS_DIR}/after + skills: ['airflow-contribution'] + + # Isolation arms + - id: anthropic:claude-agent-sdk + label: baseline + config: + <<: *base + working_dir: ${ARMS_DIR}/baseline + + - id: anthropic:claude-agent-sdk + label: skill-only + config: + <<: *base + working_dir: ${ARMS_DIR}/skill-only + skills: ['airflow-contribution'] + + - id: anthropic:claude-agent-sdk + label: agents-only + config: + <<: *base + working_dir: ${ARMS_DIR}/agents-only + +defaultTest: + options: + disableVarExpansion: true + +tests: file://cases/*.yaml diff --git a/dev/skill-evals/promptfooconfig.yaml b/dev/skill-evals/promptfooconfig.yaml new file mode 100644 index 0000000000000..d7fe4998bcc85 --- /dev/null +++ b/dev/skill-evals/promptfooconfig.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Quick mode: before vs after your change +# +# "before" = main branch version of AGENTS.md + SKILL.md +# "after" = working tree version +# +# Do not run directly — use eval.sh + +x-output-schema: &outputSchema + type: json_schema + schema: + type: object + required: [runner, command, rationale] + additionalProperties: false + properties: + runner: + type: string + enum: [uv, breeze, prek] + command: + type: string + rationale: + type: string + +x-common: &common + model: claude-sonnet-4-6 + setting_sources: ['project'] + skills: ['airflow-contribution'] + append_allowed_tools: ['Read', 'Grep', 'Glob'] + output_format: *outputSchema + +providers: + - id: anthropic:claude-agent-sdk + label: before + config: + <<: *common + working_dir: ${ARMS_DIR}/before + + - id: anthropic:claude-agent-sdk + label: after + config: + <<: *common + working_dir: ${ARMS_DIR}/after + +defaultTest: + options: + disableVarExpansion: true + assert: + - type: skill-used + value: airflow-contribution + +tests: file://cases/*.yaml From 9bb8745e1ce2c8e1cd1f169a0a00aaf70d583049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sun, 14 Jun 2026 15:09:29 +0800 Subject: [PATCH 02/16] ci: Add prek hook to remind eval on AGENTS.md and SKILL.md changes --- .pre-commit-config.yaml | 10 +++++ scripts/ci/prek/check_eval_reminder.py | 54 ++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100755 scripts/ci/prek/check_eval_reminder.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50a49acb93adb..e1e253bb44b1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1008,6 +1008,16 @@ repos: ^scripts/ci/prek/generate_agent_skills\.py$ pass_filenames: false additional_dependencies: ['pyyaml', 'rich>=13.6.0'] + - id: check-eval-reminder + name: Remind to run skill-eval on guidance changes + entry: ./scripts/ci/prek/check_eval_reminder.py + language: python + files: > + (?x) + ^(?:.*/)?AGENTS\.md$| + ^(?:.*/)?SKILL\.md$ + pass_filenames: false + verbose: true - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml language: python diff --git a/scripts/ci/prek/check_eval_reminder.py b/scripts/ci/prek/check_eval_reminder.py new file mode 100755 index 0000000000000..88fe93820dfc1 --- /dev/null +++ b/scripts/ci/prek/check_eval_reminder.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Remind contributors to run skill-eval when guidance files change. + +This hook prints a warning when AGENTS.md or SKILL.md is staged for +commit. It always exits 0 — it never blocks the commit. +""" + +from __future__ import annotations + +import subprocess +import sys + + +def get_staged_files() -> list[str]: + result = subprocess.run( + ["git", "diff", "--cached", "--name-only"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip().splitlines() + + +def main() -> int: + staged = get_staged_files() + guidance_files = [f for f in staged if f.endswith("AGENTS.md") or f.endswith("SKILL.md")] + if guidance_files: + changed = ", ".join(guidance_files) + print( + f"\n\033[33m⚠ {changed} modified — " + f"consider running ./dev/skill-evals/eval.sh before pushing\033[0m\n", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 1c6c11ca3f2ba6ee24abd23174b98c85d3aed92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Mon, 22 Jun 2026 23:05:30 +0800 Subject: [PATCH 03/16] ci: add OAuth auth and runtime config generation to skill-eval harness --- dev/skill-evals/README.md | 281 ++++++++++++++++++++++ dev/skill-evals/eval.sh | 259 +++++++++++++++----- dev/skill-evals/promptfooconfig.full.yaml | 89 ------- dev/skill-evals/promptfooconfig.yaml | 67 ------ 4 files changed, 474 insertions(+), 222 deletions(-) create mode 100644 dev/skill-evals/README.md delete mode 100644 dev/skill-evals/promptfooconfig.full.yaml delete mode 100644 dev/skill-evals/promptfooconfig.yaml diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md new file mode 100644 index 0000000000000..af7ec710e4d5f --- /dev/null +++ b/dev/skill-evals/README.md @@ -0,0 +1,281 @@ + + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Skill-Eval Harness](#skill-eval-harness) + - [Scope and limitations](#scope-and-limitations) + - [Prerequisites](#prerequisites) + - [One-time setup](#one-time-setup) + - [Quick start](#quick-start) + - [When to use which mode](#when-to-use-which-mode) + - [Usage](#usage) + - [Adding cases](#adding-cases) + - [How it works](#how-it-works) + - [Directory structure](#directory-structure) + - [Prek hook](#prek-hook) + - [Version pinning](#version-pinning) + + + TODO: This license is not consistent with the license used in the project. + Delete the inconsistent license and above line and rerun pre-commit to insert a good license. + + + +# Skill-Eval Harness + +A dev-time tool for verifying that changes to `AGENTS.md` or skill files +(`SKILL.md`) improve — or at least don't regress — agent behavior. + +The harness compares the `main` branch version against your working tree, +running the same probe cases against both and reporting the diff. Config +is generated at runtime — there are no static config files that can drift. + +## Scope and limitations + +Cases test **pure guidance decisions** — "given this scenario, which +command should the agent choose?" Arms contain only `AGENTS.md` and/or +`SKILL.md` in temporary working directories. The agent does not see +the real repository structure. + +This means the harness is suitable for: + +- Command routing decisions (uv / breeze / prek) +- Static check selection +- Guidance quality regression testing + +It is **not** suitable for cases that require the agent to explore +actual source files, read test markers, or inspect `pyproject.toml`. +If a future case needs real repo context, the arm model must change. + +## Prerequisites + +- **Node.js >=22.22.0** — check with `node --version`, upgrade with + `nvm install 22.22.0` if needed +- **`npx`** (comes with Node.js) +- **`git`** (used to extract `main` branch versions for comparison) +- **Authentication** (one of): + - Claude Code session (`claude /login`) — works with Pro/Max subscription + - `ANTHROPIC_API_KEY` environment variable — works with API credits + +## One-time setup + +Install the Claude Agent SDK in a dedicated directory (promptfoo +needs it at runtime but does not bundle it): + +```bash +mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm install @anthropic-ai/claude-agent-sdk +``` + +Log in to Claude Code (if using Pro/Max subscription): + +```bash +claude /login +``` + +## Quick start + +```bash +# Modified AGENTS.md and want to check for regressions: +./dev/skill-evals/eval.sh + +# Modified a skill and want to check for regressions vs main: +SKILL_NAME=airflow-contribution ./dev/skill-evals/eval.sh +``` + +Output: + +``` +Assembling arms ... +Mode: 2 arms, AGENTS.md only + +Changes detected (vs main): + AGENTS.md — modified + + | main | working +case-1 | PASS | PASS ← no impact +case-2 | PASS | FAIL ← regression +case-3 | FAIL | PASS ← improvement +``` + +## When to use which mode + +**Quick mode (default):** regression testing. You changed `AGENTS.md` +or `SKILL.md` and want to know "did my change make things better or +worse compared to main?" Two arms: `main` vs `working`. Fast, cheap, +answers the daily question. When used with `SKILL_NAME`, both arms +contain the skill *and* `AGENTS.md` — it tells you whether your skill +edit regressed, not whether the skill itself is effective. + +**Full mode (`--full`):** effectiveness analysis. You want to understand +*why* something passes or fails — is it the skill helping, the +`AGENTS.md` guiding, or the model already knowing? Adds isolation arms +(`baseline`, `skill-only`, `agents-only`) that separate each component's +contribution. Use periodically, not on every change. + +## Usage + +```bash +# AGENTS.md only — no skill involved +./dev/skill-evals/eval.sh + +# Skill + AGENTS.md +SKILL_NAME=airflow-contribution ./dev/skill-evals/eval.sh + +# Full research matrix (adds baseline, skill-only, agents-only) +SKILL_NAME=airflow-contribution ./dev/skill-evals/eval.sh --full + +# Use a cheaper model for fast iteration +MODEL=claude-haiku-4-5-20251001 ./dev/skill-evals/eval.sh + +# Repeat each case to account for nondeterminism (recommended: 3) +./dev/skill-evals/eval.sh --repeat 3 + +# Disable cache (force fresh LLM calls) +./dev/skill-evals/eval.sh --no-cache + +# View results in browser +npx promptfoo@0.121.17 view +``` + +### Environment variables + +| Variable | Required | Default | Description | +|----------|:--------:|---------|-------------| +| `ANTHROPIC_API_KEY` | no | — | Anthropic API key. If unset, authenticates via Claude Code session (`claude /login`) | +| `SKILL_NAME` | no | — | Skill to test (e.g. `airflow-contribution`). Omit to test AGENTS.md only | +| `MODEL` | no | `claude-sonnet-4-6` | Model to use for eval | + +### Arms by mode + +| Arm | Quick | Quick + skill | Full | Full + skill | +|-----|:-----:|:-------------:|:----:|:------------:| +| `main` | yes | yes | yes | yes | +| `working` | yes | yes | yes | yes | +| `baseline` | — | — | yes | yes | +| `agents-only` | — | — | yes | yes | +| `skill-only` | — | — | — | yes | + +`main` and `working` always contain `AGENTS.md`. When `SKILL_NAME` is +set, they also contain the skill. Isolation arms (`baseline`, +`agents-only`, `skill-only`) contain only the component named — no +other guidance files. + +`baseline` is an empty working directory. The agent receives no +`AGENTS.md` and no skill — this measures raw model capability on the +probe. Since arms contain only guidance files (no repo source code), +`baseline` tests what the model knows without any project-specific +instructions. + +## Adding cases + +Cases live in `cases/*.yaml`. Add entries to an existing file or create +a new one — no config changes required, no directory structure to learn. + +```yaml +# cases/command-routing.yaml +- description: "Provider test: uv fails, fallback to breeze" + vars: + request: | + Run amazon provider tests. + uv failed with: error: libpq-dev not found + assert: + - type: javascript + value: 'output.runner === "breeze"' +``` + +Every case runs against all arms automatically. Adding a case always +means editing exactly one file. + +The Agent SDK provider with `output_format: json_schema` returns +`output` as a parsed object — use `output.runner` directly, not +`JSON.parse(output).runner`. + +## How it works + +1. `eval.sh` extracts the `main` branch versions of `AGENTS.md` (and + `SKILL.md` if `SKILL_NAME` is set) via `git show`. If `main` does + not have the file, it falls back to the working tree copy and prints + a warning — the comparison will show no diff in that case. +2. It builds temporary working directories — one per arm — placing + files where the Claude Agent SDK discovers them (`.claude/skills/` + for skills, root for `AGENTS.md`). +3. It generates a promptfoo config into the temp directory. The output + schema is inlined in every provider block (no YAML anchors — avoids + cross-heredoc resolve issues with promptfoo's parser). +4. promptfoo runs each case against all arms in parallel via the + `anthropic:claude-agent-sdk` provider, which provides: + - `output_format: json_schema` — structured output without + markdown fence stripping + - `skill-used` assertion — confirms the agent actually invoked + the skill, not just happened to answer correctly +5. Temporary directories and config are cleaned up on exit. + +## Directory structure + +``` +dev/skill-evals/ + eval.sh # entry point — assembles arms, generates config, runs eval + README.md # this file + cases/ + command-routing.yaml # probe cases (uv / breeze / prek routing) +``` + +No static config files. The promptfoo config is generated into a temp +directory at runtime, based on `SKILL_NAME` and `--full` flags. + +## Prek hook + +A prek hook (`check-eval-reminder`) prints a reminder when `AGENTS.md` +or `SKILL.md` is staged for commit: + +``` +⚠ AGENTS.md modified — consider running ./dev/skill-evals/eval.sh before pushing +``` + +The hook never blocks the commit — it is informational only. + +## Version pinning + +The harness pins promptfoo to a specific version (`0.121.17`) to ensure +reproducible results. Update the version in `eval.sh` when upgrading. + +promptfoo is owned by OpenAI (acquired 2026-03). The harness treats it +as an execution convenience layer — the investment is in test cases +(YAML), not runner infrastructure. Cases are portable and can be run +with a simple Python script if promptfoo becomes unusable. diff --git a/dev/skill-evals/eval.sh b/dev/skill-evals/eval.sh index 4120f84fc7e48..c9f6cfec5e262 100755 --- a/dev/skill-evals/eval.sh +++ b/dev/skill-evals/eval.sh @@ -19,25 +19,54 @@ # Airflow skill-eval harness # # Usage: -# ./eval.sh Quick: 2 arms (before vs after your change) -# ./eval.sh --full Full: 4 arms (+ baseline, skill-only) -# ./eval.sh --no-cache Disable promptfoo cache -# ./eval.sh --repeat 3 Run each case N times +# ./eval.sh Test AGENTS.md changes +# SKILL_NAME=airflow-contribution ./eval.sh Test skill + AGENTS.md +# SKILL_NAME=airflow-contribution ./eval.sh --full Full research matrix +# ./eval.sh --no-cache Disable promptfoo cache +# ./eval.sh --repeat 3 Run each case N times # -# Auto-detects changes to AGENTS.md and SKILL.md against main branch. -# "before" = main's version, "after" = working tree. +# Auto-detects changes against main branch. +# "main" = main branch version +# "working" = working tree version # -# Requires: ANTHROPIC_API_KEY, npx, git +# Cases test pure guidance decisions (which command to run). Arms contain +# only AGENTS.md and/or SKILL.md — no repo source code. If a future case +# needs the agent to explore real repo structure, the arm model must change. +# +# Authentication: uses Claude Code session by default (claude /login). +# Set ANTHROPIC_API_KEY to use API key auth instead. +# +# Confirmed: output_format: json_schema makes output a parsed object, +# so assertions use output.runner directly (not JSON.parse(output).runner). set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -SKILL_SRC="$REPO_ROOT/files/agent-skill-dev/airflow-contribution/SKILL.md" AGENTS_SRC="$REPO_ROOT/AGENTS.md" -PROMPTFOO_VERSION="0.120.19" +PROMPTFOO_VERSION="0.121.17" +MODEL="${MODEL:-claude-sonnet-4-6}" + +# ── Check prerequisites ────────────────────────────────── +if ! command -v node &>/dev/null; then + echo "Error: Node.js not found. Install Node.js >=22.22.0" >&2 + exit 1 +fi +if ! command -v npx &>/dev/null; then + echo "Error: npx not found." >&2 + exit 1 +fi -# Parse --full flag, pass everything else to promptfoo +# Claude Agent SDK must be resolvable by promptfoo. Install once: +# mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk +SDK_DIR="$HOME/.promptfoo-sdk/node_modules" +if [ ! -d "$SDK_DIR/@anthropic-ai/claude-agent-sdk" ]; then + echo "Error: Claude Agent SDK not found. Run:" >&2 + echo " mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk" >&2 + exit 1 +fi + +# ── Parse flags ───────────────────────────────────────────── FULL_MODE=false PROMPTFOO_ARGS=() for arg in "$@"; do @@ -48,97 +77,195 @@ for arg in "$@"; do fi done -# Resolve base branch +# ── Resolve skill ─────────────────────────────────────────── +WITH_SKILL=false +if [ -n "${SKILL_NAME:-}" ]; then + SKILL_SRC="$REPO_ROOT/.agents/skills/$SKILL_NAME/SKILL.md" + if [ ! -f "$SKILL_SRC" ]; then + echo "Error: $SKILL_SRC not found" >&2 + exit 1 + fi + WITH_SKILL=true +fi + +# ── Resolve base branch ──────────────────────────────────── BASE_BRANCH="main" if ! git -C "$REPO_ROOT" rev-parse --verify "$BASE_BRANCH" &>/dev/null; then BASE_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)" echo "Warning: 'main' not found, using current branch '$BASE_BRANCH' as base" fi -# Create temp directory, clean up on exit -ARMS_DIR="$(mktemp -d)" -trap 'rm -rf "$ARMS_DIR"' EXIT +# ── Create temp dirs, clean up on exit ───────────────────── +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT +# Symlink node_modules so promptfoo can resolve the Claude Agent SDK +# from the temp config directory (promptfoo resolves from config basePath). +ln -s "$SDK_DIR" "$WORK_DIR/node_modules" +ARMS_DIR="$WORK_DIR/arms" +mkdir -p "$ARMS_DIR" + +# ── Extract "main" versions from git ────────────────────── +MAIN_AGENTS="$WORK_DIR/main-agents.md" +if ! git -C "$REPO_ROOT" show "$BASE_BRANCH:AGENTS.md" > "$MAIN_AGENTS" 2>/dev/null; then + cp "$AGENTS_SRC" "$MAIN_AGENTS" + echo "Warning: AGENTS.md not found on $BASE_BRANCH — main arm uses working tree copy (comparison will show no diff)" +fi + +if [ "$WITH_SKILL" = true ]; then + MAIN_SKILL="$WORK_DIR/main-skill.md" + if ! git -C "$REPO_ROOT" show "$BASE_BRANCH:.agents/skills/$SKILL_NAME/SKILL.md" > "$MAIN_SKILL" 2>/dev/null; then + cp "$SKILL_SRC" "$MAIN_SKILL" + echo "Warning: SKILL.md not found on $BASE_BRANCH — main arm uses working tree copy (comparison will show no diff)" + fi +fi -# Helper: place skill in a working dir +# ── Helpers ──────────────────────────────────────────────── place_skill() { - local dir="$1" - local source="$2" # path to SKILL.md - mkdir -p "$dir/.claude/skills/airflow-contribution" - cp "$source" "$dir/.claude/skills/airflow-contribution/SKILL.md" + mkdir -p "$1/.claude/skills/$SKILL_NAME" + cp "$2" "$1/.claude/skills/$SKILL_NAME/SKILL.md" } -# Helper: place AGENTS.md in a working dir -place_agents() { - local dir="$1" - local source="$2" # path to AGENTS.md - cp "$source" "$dir/AGENTS.md" -} +place_agents() { cp "$2" "$1/AGENTS.md"; } -# Get "before" versions from git -BEFORE_DIR="$(mktemp -d)" -trap 'rm -rf "$ARMS_DIR" "$BEFORE_DIR"' EXIT +# ── Assemble arms ───────────────────────────────────────── +echo "Assembling arms ..." -git -C "$REPO_ROOT" show "$BASE_BRANCH:AGENTS.md" > "$BEFORE_DIR/AGENTS.md" 2>/dev/null || \ - cp "$AGENTS_SRC" "$BEFORE_DIR/AGENTS.md" +mkdir -p "$ARMS_DIR/main" "$ARMS_DIR/working" -git -C "$REPO_ROOT" show "$BASE_BRANCH:files/agent-skill-dev/airflow-contribution/SKILL.md" \ - > "$BEFORE_DIR/SKILL.md" 2>/dev/null || \ - cp "$SKILL_SRC" "$BEFORE_DIR/SKILL.md" +place_agents "$ARMS_DIR/main" "$MAIN_AGENTS" +place_agents "$ARMS_DIR/working" "$AGENTS_SRC" -echo "Assembling arms ..." +if [ "$WITH_SKILL" = true ]; then + place_skill "$ARMS_DIR/main" "$MAIN_SKILL" + place_skill "$ARMS_DIR/working" "$SKILL_SRC" +fi -# === Quick mode arms (always built) === +if [ "$FULL_MODE" = true ]; then + mkdir -p "$ARMS_DIR/baseline" "$ARMS_DIR/agents-only" + place_agents "$ARMS_DIR/agents-only" "$AGENTS_SRC" -# before: main's AGENTS.md + main's skill -mkdir -p "$ARMS_DIR/before" -place_skill "$ARMS_DIR/before" "$BEFORE_DIR/SKILL.md" -place_agents "$ARMS_DIR/before" "$BEFORE_DIR/AGENTS.md" + if [ "$WITH_SKILL" = true ]; then + mkdir -p "$ARMS_DIR/skill-only" + place_skill "$ARMS_DIR/skill-only" "$SKILL_SRC" + fi +fi + +# ── Generate config ─────────────────────────────────────── +CONFIG="$WORK_DIR/promptfooconfig.yaml" +ARM_COUNT=0 + +# Output schema — inlined in every provider block (no YAML anchors, +# avoids cross-heredoc resolve issues with promptfoo's parser). +OUTPUT_SCHEMA=' output_format: + type: json_schema + schema: + type: object + required: [runner, command, rationale] + additionalProperties: false + properties: + runner: + type: string + enum: [uv, breeze, prek] + command: + type: string + rationale: + type: string' + +add_provider() { + local label="$1" + local dir="$2" + local skill="$3" # "yes" or "no" + + cat >> "$CONFIG" << EOF + - id: anthropic:claude-agent-sdk + label: $label + config: + model: $MODEL + apiKeyRequired: false + setting_sources: ['project'] + append_allowed_tools: ['Read', 'Grep', 'Glob'] + working_dir: $dir +EOF + + if [ "$skill" = "yes" ]; then + echo " skills: ['$SKILL_NAME']" >> "$CONFIG" + fi + + echo "$OUTPUT_SCHEMA" >> "$CONFIG" + echo "" >> "$CONFIG" + + ARM_COUNT=$((ARM_COUNT + 1)) +} + +# Start config — prompt template passes the request var to the agent +cat > "$CONFIG" << 'EOF' +prompts: + - '{{request}}' -# after: working tree's AGENTS.md + working tree's skill -mkdir -p "$ARMS_DIR/after" -place_skill "$ARMS_DIR/after" "$SKILL_SRC" -place_agents "$ARMS_DIR/after" "$AGENTS_SRC" +EOF +echo "providers:" >> "$CONFIG" -# === Full mode arms (only with --full) === +# Core arms +add_provider "main" "$ARMS_DIR/main" "$( [ "$WITH_SKILL" = true ] && echo yes || echo no )" +add_provider "working" "$ARMS_DIR/working" "$( [ "$WITH_SKILL" = true ] && echo yes || echo no )" + +# Full mode arms if [ "$FULL_MODE" = true ]; then - # baseline: no skill, no AGENTS.md - mkdir -p "$ARMS_DIR/baseline" + add_provider "baseline" "$ARMS_DIR/baseline" "no" + add_provider "agents-only" "$ARMS_DIR/agents-only" "no" + [ "$WITH_SKILL" = true ] && add_provider "skill-only" "$ARMS_DIR/skill-only" "yes" +fi - # skill-only: working tree's skill, no AGENTS.md - mkdir -p "$ARMS_DIR/skill-only" - place_skill "$ARMS_DIR/skill-only" "$SKILL_SRC" +# Default test config +if [ "$WITH_SKILL" = true ]; then + cat >> "$CONFIG" << EOF +defaultTest: + options: + disableVarExpansion: true + assert: + - type: skill-used + value: $SKILL_NAME - # agents-only: working tree's AGENTS.md, no skill - mkdir -p "$ARMS_DIR/agents-only" - place_agents "$ARMS_DIR/agents-only" "$AGENTS_SRC" +EOF +else + cat >> "$CONFIG" << 'EOF' +defaultTest: + options: + disableVarExpansion: true + +EOF +fi + +# Cases (single source of truth) +echo "tests: file://$SCRIPT_DIR/cases/*.yaml" >> "$CONFIG" - CONFIG="$SCRIPT_DIR/promptfooconfig.full.yaml" - echo "Mode: full (5 arms)" +# ── Report ──────────────────────────────────────────────── +if [ "$WITH_SKILL" = true ]; then + echo "Mode: $ARM_COUNT arms, skill '$SKILL_NAME'" else - CONFIG="$SCRIPT_DIR/promptfooconfig.yaml" - echo "Mode: quick (2 arms: before vs after)" + echo "Mode: $ARM_COUNT arms, AGENTS.md only" fi -# Show what changed echo "" echo "Changes detected (vs $BASE_BRANCH):" -if ! diff -q "$BEFORE_DIR/AGENTS.md" "$AGENTS_SRC" &>/dev/null; then +if ! diff -q "$MAIN_AGENTS" "$AGENTS_SRC" &>/dev/null; then echo " AGENTS.md — modified" else echo " AGENTS.md — unchanged" fi -if ! diff -q "$BEFORE_DIR/SKILL.md" "$SKILL_SRC" &>/dev/null; then - echo " SKILL.md — modified" -else - echo " SKILL.md — unchanged" +if [ "$WITH_SKILL" = true ]; then + if ! diff -q "$MAIN_SKILL" "$SKILL_SRC" &>/dev/null; then + echo " SKILL.md — modified ($SKILL_NAME)" + else + echo " SKILL.md — unchanged ($SKILL_NAME)" + fi fi echo "" -# Run -ARMS_DIR="$ARMS_DIR" \ - npx "promptfoo@$PROMPTFOO_VERSION" eval \ +# ── Run ─────────────────────────────────────────────────── +npx "promptfoo@$PROMPTFOO_VERSION" eval \ -c "$CONFIG" \ - "${PROMPTFOO_ARGS[@]}" + ${PROMPTFOO_ARGS[@]+"${PROMPTFOO_ARGS[@]}"} echo "" echo "View results: npx promptfoo@$PROMPTFOO_VERSION view" diff --git a/dev/skill-evals/promptfooconfig.full.yaml b/dev/skill-evals/promptfooconfig.full.yaml deleted file mode 100644 index bac0861808af3..0000000000000 --- a/dev/skill-evals/promptfooconfig.full.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -# Full mode: 5-arm research matrix -# -# before = main's AGENTS.md + main's SKILL.md -# after = working tree's AGENTS.md + working tree's SKILL.md -# baseline = no guidance at all -# skill-only = skill without AGENTS.md -# agents-only = AGENTS.md without skill -# -# Do not run directly — use eval.sh --full - -x-output-schema: &outputSchema - type: json_schema - schema: - type: object - required: [runner, command, rationale] - additionalProperties: false - properties: - runner: - type: string - enum: [uv, breeze, prek] - command: - type: string - rationale: - type: string - -x-base: &base - model: claude-sonnet-4-6 - setting_sources: ['project'] - append_allowed_tools: ['Read', 'Grep', 'Glob'] - output_format: *outputSchema - -providers: - # Core comparison - - id: anthropic:claude-agent-sdk - label: before - config: - <<: *base - working_dir: ${ARMS_DIR}/before - skills: ['airflow-contribution'] - - - id: anthropic:claude-agent-sdk - label: after - config: - <<: *base - working_dir: ${ARMS_DIR}/after - skills: ['airflow-contribution'] - - # Isolation arms - - id: anthropic:claude-agent-sdk - label: baseline - config: - <<: *base - working_dir: ${ARMS_DIR}/baseline - - - id: anthropic:claude-agent-sdk - label: skill-only - config: - <<: *base - working_dir: ${ARMS_DIR}/skill-only - skills: ['airflow-contribution'] - - - id: anthropic:claude-agent-sdk - label: agents-only - config: - <<: *base - working_dir: ${ARMS_DIR}/agents-only - -defaultTest: - options: - disableVarExpansion: true - -tests: file://cases/*.yaml diff --git a/dev/skill-evals/promptfooconfig.yaml b/dev/skill-evals/promptfooconfig.yaml deleted file mode 100644 index d7fe4998bcc85..0000000000000 --- a/dev/skill-evals/promptfooconfig.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -# Quick mode: before vs after your change -# -# "before" = main branch version of AGENTS.md + SKILL.md -# "after" = working tree version -# -# Do not run directly — use eval.sh - -x-output-schema: &outputSchema - type: json_schema - schema: - type: object - required: [runner, command, rationale] - additionalProperties: false - properties: - runner: - type: string - enum: [uv, breeze, prek] - command: - type: string - rationale: - type: string - -x-common: &common - model: claude-sonnet-4-6 - setting_sources: ['project'] - skills: ['airflow-contribution'] - append_allowed_tools: ['Read', 'Grep', 'Glob'] - output_format: *outputSchema - -providers: - - id: anthropic:claude-agent-sdk - label: before - config: - <<: *common - working_dir: ${ARMS_DIR}/before - - - id: anthropic:claude-agent-sdk - label: after - config: - <<: *common - working_dir: ${ARMS_DIR}/after - -defaultTest: - options: - disableVarExpansion: true - assert: - - type: skill-used - value: airflow-contribution - -tests: file://cases/*.yaml From f97d2904035d842969b55d117934e81789f99ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Mon, 29 Jun 2026 17:01:52 +0800 Subject: [PATCH 04/16] feat: add skill-eval harness for AGENTS.md regression testing --- dev/skill-evals/cases/command-routing.yaml | 35 +-- dev/skill-evals/eval.py | 299 +++++++++++++++++++++ dev/skill-evals/eval.sh | 271 ------------------- 3 files changed, 300 insertions(+), 305 deletions(-) create mode 100644 dev/skill-evals/eval.py delete mode 100755 dev/skill-evals/eval.sh diff --git a/dev/skill-evals/cases/command-routing.yaml b/dev/skill-evals/cases/command-routing.yaml index 8c6ffb6121e90..b485cac7b1dc3 100644 --- a/dev/skill-evals/cases/command-routing.yaml +++ b/dev/skill-evals/cases/command-routing.yaml @@ -15,43 +15,10 @@ # specific language governing permissions and limitations # under the License. --- -# Command routing: does the agent choose uv / breeze / prek correctly? - -- description: "Core unit test: should use uv" - vars: - request: | - Run the tests in airflow-core/tests/models/test_taskinstance.py. - The file contains @pytest.mark.db_test markers on most test classes. - The project root is airflow-core. - You are on the host, not inside a Breeze container. - assert: - - type: javascript - value: 'output.runner === "uv"' - - type: javascript - value: 'output.command.includes("uv run --project airflow-core pytest")' - -- description: "Helm test: always breeze" +- description: "Helm test: should use breeze" vars: request: | Run the Helm chart tests. - The change is in chart/templates/webserver/webserver-deployment.yaml. - Helm tests are in chart/tests/. - You are on the host, not inside a Breeze container. assert: - type: javascript value: 'output.runner === "breeze"' - - type: javascript - value: 'output.command.includes("breeze testing helm-tests")' - -- description: "mypy type check: use prek" - vars: - request: | - Run mypy type checking on the airflow-core package. - The change is in airflow-core/src/airflow/models/dag.py. - You are on the host, not inside a Breeze container. - The project uses prek hooks for static checks. - assert: - - type: javascript - value: 'output.runner === "prek"' - - type: javascript - value: 'output.command.includes("prek run mypy-airflow-core")' diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py new file mode 100644 index 0000000000000..7ecd048c37f87 --- /dev/null +++ b/dev/skill-evals/eval.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.9" +# /// +"""Airflow skill-eval harness. + +Usage: + uv run dev/skill-evals/eval.py Test AGENTS.md changes + SKILL_NAME=airflow-contribution uv run dev/skill-evals/eval.py + uv run dev/skill-evals/eval.py --full Add baseline arm + uv run dev/skill-evals/eval.py --repeat 3 --no-cache + +Arms are git worktrees of the real repo, so the agent sees actual +source files. The only difference between arms is which AGENTS.md is present. + +Authentication: uses Claude Code session by default (claude /login). +Set ANTHROPIC_API_KEY to use API key auth instead. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +PROMPTFOO_VERSION = "0.121.17" + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +SCRIPT_DIR = Path(__file__).resolve().parent +AGENTS_SRC = REPO_ROOT / "AGENTS.md" + +OUTPUT_SCHEMA = """\ + output_format: + type: json_schema + schema: + type: object + required: [runner, command, rationale] + additionalProperties: false + properties: + runner: + type: string + enum: [uv, breeze, prek] + command: + type: string + rationale: + type: string""" + + +def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True, check=False, **kwargs) + + +def check_prerequisites() -> None: + if not shutil.which("node"): + print("Error: Node.js not found. Install Node.js >=22.22.0", file=sys.stderr) + sys.exit(1) + if not shutil.which("npx"): + print("Error: npx not found.", file=sys.stderr) + sys.exit(1) + + sdk_dir = Path.home() / ".promptfoo-sdk" / "node_modules" / "@anthropic-ai" / "claude-agent-sdk" + if not sdk_dir.is_dir(): + print("Error: Claude Agent SDK not found. Run:", file=sys.stderr) + print( + " mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk" + " && npm init -y && npm install @anthropic-ai/claude-agent-sdk", + file=sys.stderr, + ) + sys.exit(1) + + +def resolve_base_branch() -> str: + result = run(["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", "main"]) + if result.returncode == 0: + return "main" + branch = run(["git", "-C", str(REPO_ROOT), "rev-parse", "--abbrev-ref", "HEAD"]).stdout.strip() + print(f"Warning: 'main' not found, using current branch '{branch}' as base") + return branch + + +def git_show_file(base_branch: str, path: str, dest: Path) -> bool: + result = run(["git", "-C", str(REPO_ROOT), "show", f"{base_branch}:{path}"]) + if result.returncode != 0: + return False + dest.write_text(result.stdout) + return True + + +def create_worktree( + work_dir: Path, + name: str, + base_branch: str, + agents_file: Path | None, + skill_name: str | None = None, + skill_file: Path | None = None, +) -> Path: + """Create a git worktree arm with the specified AGENTS.md and optional SKILL.md.""" + wt_dir = work_dir / name + run( + [ + "git", + "-C", + str(REPO_ROOT), + "worktree", + "add", + "--quiet", + "--detach", + str(wt_dir), + base_branch, + ] + ) + + if agents_file is None: + (wt_dir / "AGENTS.md").unlink(missing_ok=True) + else: + shutil.copy2(agents_file, wt_dir / "AGENTS.md") + + if skill_name and skill_file: + skill_dir = wt_dir / ".agents" / "skills" / skill_name + skill_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(skill_file, skill_dir / "SKILL.md") + + return wt_dir + + +def remove_worktree(wt_dir: Path) -> None: + run(["git", "-C", str(REPO_ROOT), "worktree", "remove", "--force", str(wt_dir)]) + + +def add_provider( + config_lines: list[str], + label: str, + working_dir: Path, + model: str, + skill_name: str | None = None, +) -> None: + config_lines.append(" - id: anthropic:claude-agent-sdk") + config_lines.append(f" label: {label}") + config_lines.append(" config:") + config_lines.append(f" model: {model}") + config_lines.append(" apiKeyRequired: false") + config_lines.append(" setting_sources: ['project']") + config_lines.append(" append_allowed_tools: ['Read', 'Grep', 'Glob']") + config_lines.append(f" working_dir: {working_dir}") + if skill_name: + config_lines.append(f" skills: ['{skill_name}']") + config_lines.append(OUTPUT_SCHEMA) + config_lines.append("") + + +def main() -> int: + check_prerequisites() + + model = os.environ.get("MODEL", "claude-sonnet-4-6") + skill_name = os.environ.get("SKILL_NAME") + skill_src = None + if skill_name: + skill_src = REPO_ROOT / ".agents" / "skills" / skill_name / "SKILL.md" + if not skill_src.is_file(): + print(f"Error: {skill_src} not found", file=sys.stderr) + return 1 + + # Parse flags + full_mode = "--full" in sys.argv + promptfoo_args = [a for a in sys.argv[1:] if a != "--full"] + + base_branch = resolve_base_branch() + + # Temp dir for config and worktrees + work_dir = Path(tempfile.mkdtemp()) + worktrees: list[Path] = [] + + try: + # Symlink node_modules for promptfoo SDK resolution + sdk_modules = Path.home() / ".promptfoo-sdk" / "node_modules" + (work_dir / "node_modules").symlink_to(sdk_modules) + + # Extract main-branch AGENTS.md + main_agents = work_dir / "main-agents.md" + if not git_show_file(base_branch, "AGENTS.md", main_agents): + shutil.copy2(AGENTS_SRC, main_agents) + print(f"Warning: AGENTS.md not found on {base_branch} — main arm uses working tree copy") + + main_skill = None + if skill_name and skill_src: + main_skill = work_dir / "main-skill.md" + skill_git_path = f".agents/skills/{skill_name}/SKILL.md" + if not git_show_file(base_branch, skill_git_path, main_skill): + shutil.copy2(skill_src, main_skill) + print(f"Warning: SKILL.md not found on {base_branch} — main arm uses working tree copy") + + # Assemble arms + print("Assembling arms (git worktrees) ...") + + arm_main = create_worktree(work_dir, "main", base_branch, main_agents, skill_name, main_skill) + worktrees.append(arm_main) + + arm_working = create_worktree(work_dir, "working", base_branch, AGENTS_SRC, skill_name, skill_src) + worktrees.append(arm_working) + + arm_baseline = None + if full_mode: + arm_baseline = create_worktree(work_dir, "baseline", base_branch, None) + worktrees.append(arm_baseline) + + # Generate config + config_lines: list[str] = [] + config_lines.append("prompts:") + config_lines.append(" - '{{request}}'") + config_lines.append("") + config_lines.append("providers:") + + arm_count = 0 + skill_for_provider = skill_name if skill_name else None + + add_provider(config_lines, "main", arm_main, model, skill_for_provider) + arm_count += 1 + add_provider(config_lines, "working", arm_working, model, skill_for_provider) + arm_count += 1 + + if full_mode and arm_baseline: + add_provider(config_lines, "baseline", arm_baseline, model) + arm_count += 1 + + # Default test config + config_lines.append("defaultTest:") + config_lines.append(" options:") + config_lines.append(" disableVarExpansion: true") + if skill_name: + config_lines.append(" assert:") + config_lines.append(" - type: skill-used") + config_lines.append(f" value: {skill_name}") + config_lines.append("") + + # Cases + config_lines.append(f"tests: file://{SCRIPT_DIR}/cases/*.yaml") + + config_path = work_dir / "promptfooconfig.yaml" + config_path.write_text("\n".join(config_lines) + "\n") + + # Report + if skill_name: + print(f"Mode: {arm_count} arms, skill '{skill_name}'") + else: + print(f"Mode: {arm_count} arms, AGENTS.md only") + + print() + print(f"Changes detected (vs {base_branch}):") + agents_diff = run(["diff", "-q", str(main_agents), str(AGENTS_SRC)]) + if agents_diff.returncode != 0: + print(" AGENTS.md — modified") + else: + print(" AGENTS.md — unchanged") + + if skill_name and main_skill: + skill_diff = run(["diff", "-q", str(main_skill), str(skill_src)]) + if skill_diff.returncode != 0: + print(f" SKILL.md — modified ({skill_name})") + else: + print(f" SKILL.md — unchanged ({skill_name})") + print() + + # Run promptfoo + result = subprocess.run( + ["npx", f"promptfoo@{PROMPTFOO_VERSION}", "eval", "-c", str(config_path), *promptfoo_args], + check=False, + ) + + print() + print(f"View results: npx promptfoo@{PROMPTFOO_VERSION} view") + return result.returncode + + finally: + for wt in worktrees: + remove_worktree(wt) + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dev/skill-evals/eval.sh b/dev/skill-evals/eval.sh deleted file mode 100755 index c9f6cfec5e262..0000000000000 --- a/dev/skill-evals/eval.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Airflow skill-eval harness -# -# Usage: -# ./eval.sh Test AGENTS.md changes -# SKILL_NAME=airflow-contribution ./eval.sh Test skill + AGENTS.md -# SKILL_NAME=airflow-contribution ./eval.sh --full Full research matrix -# ./eval.sh --no-cache Disable promptfoo cache -# ./eval.sh --repeat 3 Run each case N times -# -# Auto-detects changes against main branch. -# "main" = main branch version -# "working" = working tree version -# -# Cases test pure guidance decisions (which command to run). Arms contain -# only AGENTS.md and/or SKILL.md — no repo source code. If a future case -# needs the agent to explore real repo structure, the arm model must change. -# -# Authentication: uses Claude Code session by default (claude /login). -# Set ANTHROPIC_API_KEY to use API key auth instead. -# -# Confirmed: output_format: json_schema makes output a parsed object, -# so assertions use output.runner directly (not JSON.parse(output).runner). - -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -AGENTS_SRC="$REPO_ROOT/AGENTS.md" -PROMPTFOO_VERSION="0.121.17" -MODEL="${MODEL:-claude-sonnet-4-6}" - -# ── Check prerequisites ────────────────────────────────── -if ! command -v node &>/dev/null; then - echo "Error: Node.js not found. Install Node.js >=22.22.0" >&2 - exit 1 -fi -if ! command -v npx &>/dev/null; then - echo "Error: npx not found." >&2 - exit 1 -fi - -# Claude Agent SDK must be resolvable by promptfoo. Install once: -# mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk -SDK_DIR="$HOME/.promptfoo-sdk/node_modules" -if [ ! -d "$SDK_DIR/@anthropic-ai/claude-agent-sdk" ]; then - echo "Error: Claude Agent SDK not found. Run:" >&2 - echo " mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk" >&2 - exit 1 -fi - -# ── Parse flags ───────────────────────────────────────────── -FULL_MODE=false -PROMPTFOO_ARGS=() -for arg in "$@"; do - if [ "$arg" = "--full" ]; then - FULL_MODE=true - else - PROMPTFOO_ARGS+=("$arg") - fi -done - -# ── Resolve skill ─────────────────────────────────────────── -WITH_SKILL=false -if [ -n "${SKILL_NAME:-}" ]; then - SKILL_SRC="$REPO_ROOT/.agents/skills/$SKILL_NAME/SKILL.md" - if [ ! -f "$SKILL_SRC" ]; then - echo "Error: $SKILL_SRC not found" >&2 - exit 1 - fi - WITH_SKILL=true -fi - -# ── Resolve base branch ──────────────────────────────────── -BASE_BRANCH="main" -if ! git -C "$REPO_ROOT" rev-parse --verify "$BASE_BRANCH" &>/dev/null; then - BASE_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)" - echo "Warning: 'main' not found, using current branch '$BASE_BRANCH' as base" -fi - -# ── Create temp dirs, clean up on exit ───────────────────── -WORK_DIR="$(mktemp -d)" -trap 'rm -rf "$WORK_DIR"' EXIT -# Symlink node_modules so promptfoo can resolve the Claude Agent SDK -# from the temp config directory (promptfoo resolves from config basePath). -ln -s "$SDK_DIR" "$WORK_DIR/node_modules" -ARMS_DIR="$WORK_DIR/arms" -mkdir -p "$ARMS_DIR" - -# ── Extract "main" versions from git ────────────────────── -MAIN_AGENTS="$WORK_DIR/main-agents.md" -if ! git -C "$REPO_ROOT" show "$BASE_BRANCH:AGENTS.md" > "$MAIN_AGENTS" 2>/dev/null; then - cp "$AGENTS_SRC" "$MAIN_AGENTS" - echo "Warning: AGENTS.md not found on $BASE_BRANCH — main arm uses working tree copy (comparison will show no diff)" -fi - -if [ "$WITH_SKILL" = true ]; then - MAIN_SKILL="$WORK_DIR/main-skill.md" - if ! git -C "$REPO_ROOT" show "$BASE_BRANCH:.agents/skills/$SKILL_NAME/SKILL.md" > "$MAIN_SKILL" 2>/dev/null; then - cp "$SKILL_SRC" "$MAIN_SKILL" - echo "Warning: SKILL.md not found on $BASE_BRANCH — main arm uses working tree copy (comparison will show no diff)" - fi -fi - -# ── Helpers ──────────────────────────────────────────────── -place_skill() { - mkdir -p "$1/.claude/skills/$SKILL_NAME" - cp "$2" "$1/.claude/skills/$SKILL_NAME/SKILL.md" -} - -place_agents() { cp "$2" "$1/AGENTS.md"; } - -# ── Assemble arms ───────────────────────────────────────── -echo "Assembling arms ..." - -mkdir -p "$ARMS_DIR/main" "$ARMS_DIR/working" - -place_agents "$ARMS_DIR/main" "$MAIN_AGENTS" -place_agents "$ARMS_DIR/working" "$AGENTS_SRC" - -if [ "$WITH_SKILL" = true ]; then - place_skill "$ARMS_DIR/main" "$MAIN_SKILL" - place_skill "$ARMS_DIR/working" "$SKILL_SRC" -fi - -if [ "$FULL_MODE" = true ]; then - mkdir -p "$ARMS_DIR/baseline" "$ARMS_DIR/agents-only" - place_agents "$ARMS_DIR/agents-only" "$AGENTS_SRC" - - if [ "$WITH_SKILL" = true ]; then - mkdir -p "$ARMS_DIR/skill-only" - place_skill "$ARMS_DIR/skill-only" "$SKILL_SRC" - fi -fi - -# ── Generate config ─────────────────────────────────────── -CONFIG="$WORK_DIR/promptfooconfig.yaml" -ARM_COUNT=0 - -# Output schema — inlined in every provider block (no YAML anchors, -# avoids cross-heredoc resolve issues with promptfoo's parser). -OUTPUT_SCHEMA=' output_format: - type: json_schema - schema: - type: object - required: [runner, command, rationale] - additionalProperties: false - properties: - runner: - type: string - enum: [uv, breeze, prek] - command: - type: string - rationale: - type: string' - -add_provider() { - local label="$1" - local dir="$2" - local skill="$3" # "yes" or "no" - - cat >> "$CONFIG" << EOF - - id: anthropic:claude-agent-sdk - label: $label - config: - model: $MODEL - apiKeyRequired: false - setting_sources: ['project'] - append_allowed_tools: ['Read', 'Grep', 'Glob'] - working_dir: $dir -EOF - - if [ "$skill" = "yes" ]; then - echo " skills: ['$SKILL_NAME']" >> "$CONFIG" - fi - - echo "$OUTPUT_SCHEMA" >> "$CONFIG" - echo "" >> "$CONFIG" - - ARM_COUNT=$((ARM_COUNT + 1)) -} - -# Start config — prompt template passes the request var to the agent -cat > "$CONFIG" << 'EOF' -prompts: - - '{{request}}' - -EOF -echo "providers:" >> "$CONFIG" - -# Core arms -add_provider "main" "$ARMS_DIR/main" "$( [ "$WITH_SKILL" = true ] && echo yes || echo no )" -add_provider "working" "$ARMS_DIR/working" "$( [ "$WITH_SKILL" = true ] && echo yes || echo no )" - -# Full mode arms -if [ "$FULL_MODE" = true ]; then - add_provider "baseline" "$ARMS_DIR/baseline" "no" - add_provider "agents-only" "$ARMS_DIR/agents-only" "no" - [ "$WITH_SKILL" = true ] && add_provider "skill-only" "$ARMS_DIR/skill-only" "yes" -fi - -# Default test config -if [ "$WITH_SKILL" = true ]; then - cat >> "$CONFIG" << EOF -defaultTest: - options: - disableVarExpansion: true - assert: - - type: skill-used - value: $SKILL_NAME - -EOF -else - cat >> "$CONFIG" << 'EOF' -defaultTest: - options: - disableVarExpansion: true - -EOF -fi - -# Cases (single source of truth) -echo "tests: file://$SCRIPT_DIR/cases/*.yaml" >> "$CONFIG" - -# ── Report ──────────────────────────────────────────────── -if [ "$WITH_SKILL" = true ]; then - echo "Mode: $ARM_COUNT arms, skill '$SKILL_NAME'" -else - echo "Mode: $ARM_COUNT arms, AGENTS.md only" -fi - -echo "" -echo "Changes detected (vs $BASE_BRANCH):" -if ! diff -q "$MAIN_AGENTS" "$AGENTS_SRC" &>/dev/null; then - echo " AGENTS.md — modified" -else - echo " AGENTS.md — unchanged" -fi -if [ "$WITH_SKILL" = true ]; then - if ! diff -q "$MAIN_SKILL" "$SKILL_SRC" &>/dev/null; then - echo " SKILL.md — modified ($SKILL_NAME)" - else - echo " SKILL.md — unchanged ($SKILL_NAME)" - fi -fi -echo "" - -# ── Run ─────────────────────────────────────────────────── -npx "promptfoo@$PROMPTFOO_VERSION" eval \ - -c "$CONFIG" \ - ${PROMPTFOO_ARGS[@]+"${PROMPTFOO_ARGS[@]}"} - -echo "" -echo "View results: npx promptfoo@$PROMPTFOO_VERSION view" From b4542ec97f0cc4e3fe467726c67c2333d2e06f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Mon, 29 Jun 2026 17:34:19 +0800 Subject: [PATCH 05/16] docs: update skill-eval README and use Helm routing as starter case --- dev/skill-evals/README.md | 243 +++++++------------------------------- 1 file changed, 42 insertions(+), 201 deletions(-) diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index af7ec710e4d5f..65e23c1b33668 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -15,267 +15,108 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - --> +--> **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Skill-Eval Harness](#skill-eval-harness) - - [Scope and limitations](#scope-and-limitations) - [Prerequisites](#prerequisites) - [One-time setup](#one-time-setup) - - [Quick start](#quick-start) - - [When to use which mode](#when-to-use-which-mode) - [Usage](#usage) - [Adding cases](#adding-cases) - [How it works](#how-it-works) - - [Directory structure](#directory-structure) - [Prek hook](#prek-hook) - - [Version pinning](#version-pinning) - TODO: This license is not consistent with the license used in the project. - Delete the inconsistent license and above line and rerun pre-commit to insert a good license. - - # Skill-Eval Harness -A dev-time tool for verifying that changes to `AGENTS.md` or skill files -(`SKILL.md`) improve — or at least don't regress — agent behavior. - -The harness compares the `main` branch version against your working tree, -running the same probe cases against both and reporting the diff. Config -is generated at runtime — there are no static config files that can drift. - -## Scope and limitations +Test whether changes to `AGENTS.md` break or improve agent routing +decisions. The harness compares the `main` branch version against +your working tree, running the same cases against both and reporting +the diff. -Cases test **pure guidance decisions** — "given this scenario, which -command should the agent choose?" Arms contain only `AGENTS.md` and/or -`SKILL.md` in temporary working directories. The agent does not see -the real repository structure. - -This means the harness is suitable for: - -- Command routing decisions (uv / breeze / prek) -- Static check selection -- Guidance quality regression testing - -It is **not** suitable for cases that require the agent to explore -actual source files, read test markers, or inspect `pyproject.toml`. -If a future case needs real repo context, the arm model must change. +Each arm is a **git worktree** of the real repo — the agent sees +actual source files (`pyproject.toml`, directory structure, etc.). +The only difference between arms is which `AGENTS.md` is present. ## Prerequisites -- **Node.js >=22.22.0** — check with `node --version`, upgrade with - `nvm install 22.22.0` if needed -- **`npx`** (comes with Node.js) -- **`git`** (used to extract `main` branch versions for comparison) +- **Node.js >=22.22.0** — check with `node --version` - **Authentication** (one of): - - Claude Code session (`claude /login`) — works with Pro/Max subscription - - `ANTHROPIC_API_KEY` environment variable — works with API credits + - Claude Code session (`claude /login`) — Pro/Max subscription + - `ANTHROPIC_API_KEY` environment variable — API credits ## One-time setup -Install the Claude Agent SDK in a dedicated directory (promptfoo -needs it at runtime but does not bundle it): - -```bash -mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm install @anthropic-ai/claude-agent-sdk -``` - -Log in to Claude Code (if using Pro/Max subscription): +Install the Claude Agent SDK (promptfoo needs it at runtime): ```bash -claude /login +mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk ``` -## Quick start - -```bash -# Modified AGENTS.md and want to check for regressions: -./dev/skill-evals/eval.sh - -# Modified a skill and want to check for regressions vs main: -SKILL_NAME=airflow-contribution ./dev/skill-evals/eval.sh -``` - -Output: - -``` -Assembling arms ... -Mode: 2 arms, AGENTS.md only - -Changes detected (vs main): - AGENTS.md — modified - - | main | working -case-1 | PASS | PASS ← no impact -case-2 | PASS | FAIL ← regression -case-3 | FAIL | PASS ← improvement -``` - -## When to use which mode - -**Quick mode (default):** regression testing. You changed `AGENTS.md` -or `SKILL.md` and want to know "did my change make things better or -worse compared to main?" Two arms: `main` vs `working`. Fast, cheap, -answers the daily question. When used with `SKILL_NAME`, both arms -contain the skill *and* `AGENTS.md` — it tells you whether your skill -edit regressed, not whether the skill itself is effective. - -**Full mode (`--full`):** effectiveness analysis. You want to understand -*why* something passes or fails — is it the skill helping, the -`AGENTS.md` guiding, or the model already knowing? Adds isolation arms -(`baseline`, `skill-only`, `agents-only`) that separate each component's -contribution. Use periodically, not on every change. - ## Usage ```bash -# AGENTS.md only — no skill involved -./dev/skill-evals/eval.sh +# Check for regressions after editing AGENTS.md: +uv run dev/skill-evals/eval.py -# Skill + AGENTS.md -SKILL_NAME=airflow-contribution ./dev/skill-evals/eval.sh +# Repeat each case to reduce nondeterminism: +uv run dev/skill-evals/eval.py --repeat 3 -# Full research matrix (adds baseline, skill-only, agents-only) -SKILL_NAME=airflow-contribution ./dev/skill-evals/eval.sh --full +# Add baseline arm (no AGENTS.md) to measure raw model capability: +uv run dev/skill-evals/eval.py --full -# Use a cheaper model for fast iteration -MODEL=claude-haiku-4-5-20251001 ./dev/skill-evals/eval.sh +# Test a skill alongside AGENTS.md: +SKILL_NAME=airflow-contribution uv run dev/skill-evals/eval.py -# Repeat each case to account for nondeterminism (recommended: 3) -./dev/skill-evals/eval.sh --repeat 3 +# Use a cheaper model for fast iteration: +MODEL=claude-haiku-4-5-20251001 uv run dev/skill-evals/eval.py -# Disable cache (force fresh LLM calls) -./dev/skill-evals/eval.sh --no-cache +# Disable cache (force fresh LLM calls): +uv run dev/skill-evals/eval.py --no-cache -# View results in browser +# View results in browser: npx promptfoo@0.121.17 view ``` -### Environment variables - -| Variable | Required | Default | Description | -|----------|:--------:|---------|-------------| -| `ANTHROPIC_API_KEY` | no | — | Anthropic API key. If unset, authenticates via Claude Code session (`claude /login`) | -| `SKILL_NAME` | no | — | Skill to test (e.g. `airflow-contribution`). Omit to test AGENTS.md only | -| `MODEL` | no | `claude-sonnet-4-6` | Model to use for eval | - -### Arms by mode - -| Arm | Quick | Quick + skill | Full | Full + skill | -|-----|:-----:|:-------------:|:----:|:------------:| -| `main` | yes | yes | yes | yes | -| `working` | yes | yes | yes | yes | -| `baseline` | — | — | yes | yes | -| `agents-only` | — | — | yes | yes | -| `skill-only` | — | — | — | yes | - -`main` and `working` always contain `AGENTS.md`. When `SKILL_NAME` is -set, they also contain the skill. Isolation arms (`baseline`, -`agents-only`, `skill-only`) contain only the component named — no -other guidance files. - -`baseline` is an empty working directory. The agent receives no -`AGENTS.md` and no skill — this measures raw model capability on the -probe. Since arms contain only guidance files (no repo source code), -`baseline` tests what the model knows without any project-specific -instructions. - ## Adding cases -Cases live in `cases/*.yaml`. Add entries to an existing file or create -a new one — no config changes required, no directory structure to learn. +Cases live in `cases/*.yaml`. Add entries to an existing file or +create a new one — no config changes needed. ```yaml -# cases/command-routing.yaml -- description: "Provider test: uv fails, fallback to breeze" +- description: "Helm test: should use breeze" vars: request: | - Run amazon provider tests. - uv failed with: error: libpq-dev not found + Run the Helm chart tests. assert: - type: javascript value: 'output.runner === "breeze"' ``` -Every case runs against all arms automatically. Adding a case always -means editing exactly one file. - -The Agent SDK provider with `output_format: json_schema` returns -`output` as a parsed object — use `output.runner` directly, not -`JSON.parse(output).runner`. +The agent returns structured JSON (`{runner, command, rationale}`). +Use `output.runner` directly in assertions. ## How it works -1. `eval.sh` extracts the `main` branch versions of `AGENTS.md` (and - `SKILL.md` if `SKILL_NAME` is set) via `git show`. If `main` does - not have the file, it falls back to the working tree copy and prints - a warning — the comparison will show no diff in that case. -2. It builds temporary working directories — one per arm — placing - files where the Claude Agent SDK discovers them (`.claude/skills/` - for skills, root for `AGENTS.md`). -3. It generates a promptfoo config into the temp directory. The output - schema is inlined in every provider block (no YAML anchors — avoids - cross-heredoc resolve issues with promptfoo's parser). -4. promptfoo runs each case against all arms in parallel via the - `anthropic:claude-agent-sdk` provider, which provides: - - `output_format: json_schema` — structured output without - markdown fence stripping - - `skill-used` assertion — confirms the agent actually invoked - the skill, not just happened to answer correctly -5. Temporary directories and config are cleaned up on exit. - -## Directory structure - -``` -dev/skill-evals/ - eval.sh # entry point — assembles arms, generates config, runs eval - README.md # this file - cases/ - command-routing.yaml # probe cases (uv / breeze / prek routing) -``` - -No static config files. The promptfoo config is generated into a temp -directory at runtime, based on `SKILL_NAME` and `--full` flags. +1. Creates git worktrees — one with `main`'s AGENTS.md, one with + your working tree version. Both are full repo checkouts. +2. Generates a [promptfoo](https://github.com/promptfoo/promptfoo) + config with `anthropic:claude-agent-sdk` provider and + `output_format: json_schema` for structured output. +3. Runs each case against all arms in parallel. +4. Reports pass/fail diff. Worktrees cleaned up on exit. ## Prek hook -A prek hook (`check-eval-reminder`) prints a reminder when `AGENTS.md` -or `SKILL.md` is staged for commit: +A prek hook prints a reminder when `AGENTS.md` or `SKILL.md` is +staged for commit: ``` -⚠ AGENTS.md modified — consider running ./dev/skill-evals/eval.sh before pushing +⚠ AGENTS.md modified — consider running uv run dev/skill-evals/eval.py before pushing ``` The hook never blocks the commit — it is informational only. - -## Version pinning - -The harness pins promptfoo to a specific version (`0.121.17`) to ensure -reproducible results. Update the version in `eval.sh` when upgrading. - -promptfoo is owned by OpenAI (acquired 2026-03). The harness treats it -as an execution convenience layer — the investment is in test cases -(YAML), not runner infrastructure. Cases are portable and can be run -with a simple Python script if promptfoo becomes unusable. From ef761c6722d6f615c60b9fe4eb409bc79ee011ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Tue, 30 Jun 2026 00:09:20 +0800 Subject: [PATCH 06/16] fix: point skill-eval reminder hook at eval.py, not nonexistent eval.sh --- scripts/ci/prek/check_eval_reminder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/prek/check_eval_reminder.py b/scripts/ci/prek/check_eval_reminder.py index 88fe93820dfc1..1d04dadd27346 100755 --- a/scripts/ci/prek/check_eval_reminder.py +++ b/scripts/ci/prek/check_eval_reminder.py @@ -44,7 +44,7 @@ def main() -> int: changed = ", ".join(guidance_files) print( f"\n\033[33m⚠ {changed} modified — " - f"consider running ./dev/skill-evals/eval.sh before pushing\033[0m\n", + f"consider running 'uv run dev/skill-evals/eval.py' before pushing\033[0m\n", file=sys.stderr, ) return 0 From 516163e5d4aa9f34117dcf11ad0eb9f156dc7159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Tue, 30 Jun 2026 00:20:21 +0800 Subject: [PATCH 07/16] refactor: skip working arm when unchanged, show model in output --- dev/skill-evals/eval.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index 7ecd048c37f87..8e4f5dac32dc2 100644 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -208,14 +208,25 @@ def main() -> int: shutil.copy2(skill_src, main_skill) print(f"Warning: SKILL.md not found on {base_branch} — main arm uses working tree copy") + # Detect changes to decide which arms to build + agents_changed = run(["diff", "-q", str(main_agents), str(AGENTS_SRC)]).returncode != 0 + skill_changed = False + if skill_name and main_skill and skill_src: + skill_changed = run(["diff", "-q", str(main_skill), str(skill_src)]).returncode != 0 + need_working = agents_changed or skill_changed + # Assemble arms print("Assembling arms (git worktrees) ...") arm_main = create_worktree(work_dir, "main", base_branch, main_agents, skill_name, main_skill) worktrees.append(arm_main) - arm_working = create_worktree(work_dir, "working", base_branch, AGENTS_SRC, skill_name, skill_src) - worktrees.append(arm_working) + arm_working = None + if need_working: + arm_working = create_worktree(work_dir, "working", base_branch, AGENTS_SRC, skill_name, skill_src) + worktrees.append(arm_working) + else: + print(" AGENTS.md unchanged — skipping working arm") arm_baseline = None if full_mode: @@ -234,8 +245,10 @@ def main() -> int: add_provider(config_lines, "main", arm_main, model, skill_for_provider) arm_count += 1 - add_provider(config_lines, "working", arm_working, model, skill_for_provider) - arm_count += 1 + + if arm_working: + add_provider(config_lines, "working", arm_working, model, skill_for_provider) + arm_count += 1 if full_mode and arm_baseline: add_provider(config_lines, "baseline", arm_baseline, model) @@ -259,24 +272,15 @@ def main() -> int: # Report if skill_name: - print(f"Mode: {arm_count} arms, skill '{skill_name}'") + print(f"Mode: {arm_count} arms, skill '{skill_name}', model: {model}") else: - print(f"Mode: {arm_count} arms, AGENTS.md only") + print(f"Mode: {arm_count} arms, AGENTS.md only, model: {model}") print() print(f"Changes detected (vs {base_branch}):") - agents_diff = run(["diff", "-q", str(main_agents), str(AGENTS_SRC)]) - if agents_diff.returncode != 0: - print(" AGENTS.md — modified") - else: - print(" AGENTS.md — unchanged") - - if skill_name and main_skill: - skill_diff = run(["diff", "-q", str(main_skill), str(skill_src)]) - if skill_diff.returncode != 0: - print(f" SKILL.md — modified ({skill_name})") - else: - print(f" SKILL.md — unchanged ({skill_name})") + print(f" AGENTS.md — {'modified' if agents_changed else 'unchanged'}") + if skill_name: + print(f" SKILL.md — {'modified' if skill_changed else 'unchanged'} ({skill_name})") print() # Run promptfoo From 45b9eb4b9b2385dfe968b97b562b5a3c30337121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Tue, 30 Jun 2026 00:22:15 +0800 Subject: [PATCH 08/16] refactor: skip working arm when unchanged, show model, use promptfoo@latest --- dev/skill-evals/README.md | 2 +- dev/skill-evals/eval.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 65e23c1b33668..56b9fd947e672 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -79,7 +79,7 @@ MODEL=claude-haiku-4-5-20251001 uv run dev/skill-evals/eval.py uv run dev/skill-evals/eval.py --no-cache # View results in browser: -npx promptfoo@0.121.17 view +npx promptfoo@latest view ``` ## Adding cases diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index 8e4f5dac32dc2..fae83e1d0500d 100644 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -42,7 +42,7 @@ import tempfile from pathlib import Path -PROMPTFOO_VERSION = "0.121.17" +PROMPTFOO_VERSION = "latest" REPO_ROOT = Path(__file__).resolve().parent.parent.parent SCRIPT_DIR = Path(__file__).resolve().parent From e3dded0e056af23784a65cc7467e693411ee00bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Thu, 2 Jul 2026 13:22:24 +0800 Subject: [PATCH 09/16] refactor: replace command-routing cases with newsfragment cases from real PRs --- dev/skill-evals/README.md | 19 ++++--- dev/skill-evals/cases/command-routing.yaml | 24 -------- dev/skill-evals/cases/newsfragment.yaml | 64 ++++++++++++++++++++++ dev/skill-evals/eval.py | 27 +++++---- 4 files changed, 87 insertions(+), 47 deletions(-) delete mode 100644 dev/skill-evals/cases/command-routing.yaml create mode 100644 dev/skill-evals/cases/newsfragment.yaml diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 56b9fd947e672..5cd1bef3e39da 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -33,10 +33,9 @@ # Skill-Eval Harness -Test whether changes to `AGENTS.md` break or improve agent routing -decisions. The harness compares the `main` branch version against -your working tree, running the same cases against both and reporting -the diff. +Test whether changes to `AGENTS.md` break or improve agent decisions. +The harness compares the `main` branch version against your working +tree, running the same cases against both and reporting the diff. Each arm is a **git worktree** of the real repo — the agent sees actual source files (`pyproject.toml`, directory structure, etc.). @@ -88,17 +87,19 @@ Cases live in `cases/*.yaml`. Add entries to an existing file or create a new one — no config changes needed. ```yaml -- description: "Helm test: should use breeze" +- description: "Scheduler bugfix (#64322): no newsfragment" vars: request: | - Run the Helm chart tests. + I fixed the scheduler to skip asset-triggered Dags that don't + have a SerializedDagModel yet. + Should I create a newsfragment? assert: - type: javascript - value: 'output.runner === "breeze"' + value: 'output.should_create === false' ``` -The agent returns structured JSON (`{runner, command, rationale}`). -Use `output.runner` directly in assertions. +The agent returns structured JSON (`{should_create, type, rationale}`). +Use `output.should_create` directly in assertions. ## How it works diff --git a/dev/skill-evals/cases/command-routing.yaml b/dev/skill-evals/cases/command-routing.yaml deleted file mode 100644 index b485cac7b1dc3..0000000000000 --- a/dev/skill-evals/cases/command-routing.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -- description: "Helm test: should use breeze" - vars: - request: | - Run the Helm chart tests. - assert: - - type: javascript - value: 'output.runner === "breeze"' diff --git a/dev/skill-evals/cases/newsfragment.yaml b/dev/skill-evals/cases/newsfragment.yaml new file mode 100644 index 0000000000000..7b0d4e630fa26 --- /dev/null +++ b/dev/skill-evals/cases/newsfragment.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Cases from real PRs where agents created newsfragments that +# reviewers asked to remove. These are the ambiguous boundary +# cases — the changes look like they could be user-facing. + +- description: "Scheduler bugfix (#64322): no newsfragment" + vars: + request: | + I fixed the scheduler to skip asset-triggered Dags that don't + have a SerializedDagModel yet. Without this fix, premature + asset-triggered DagRuns could be created during parsing. + The fix is in airflow-core/src/airflow/models/dag.py. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "i18n cache busting (#65720): no newsfragment" + vars: + request: | + I fixed stale translation files after Airflow upgrades by + appending the version number to the i18n file load path. + The fix is in airflow-core/src/airflow/ui/src/i18n/config.ts. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "API query optimization (#66696): no newsfragment" + vars: + request: | + I replaced COALESCE with index-friendly OR conditions in the + datetime range filters in airflow-core API to improve query + performance. No behavior change for API consumers. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "Provider bug fix (#67333): no newsfragment" + vars: + request: | + I fixed a monitoring-pod leak in KubernetesJobOperator. + The fix is in providers/cncf/kubernetes/. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index fae83e1d0500d..d0f48908bf378 100644 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -20,17 +20,16 @@ # /// """Airflow skill-eval harness. -Usage: - uv run dev/skill-evals/eval.py Test AGENTS.md changes - SKILL_NAME=airflow-contribution uv run dev/skill-evals/eval.py - uv run dev/skill-evals/eval.py --full Add baseline arm - uv run dev/skill-evals/eval.py --repeat 3 --no-cache +Test whether AGENTS.md guidance affects agent decisions by comparing +arms with and without the guidance. Each arm is a git worktree of the +real repo — the agent sees actual source files. -Arms are git worktrees of the real repo, so the agent sees actual -source files. The only difference between arms is which AGENTS.md is present. +Usage: + uv run dev/skill-evals/eval.py Test AGENTS.md changes + uv run dev/skill-evals/eval.py --full Add baseline arm (no AGENTS.md) + uv run dev/skill-evals/eval.py --repeat 3 Reduce nondeterminism -Authentication: uses Claude Code session by default (claude /login). -Set ANTHROPIC_API_KEY to use API key auth instead. +Authentication: Claude Code session (claude /login) or ANTHROPIC_API_KEY. """ from __future__ import annotations @@ -53,14 +52,14 @@ type: json_schema schema: type: object - required: [runner, command, rationale] + required: [should_create, rationale] additionalProperties: false properties: - runner: - type: string - enum: [uv, breeze, prek] - command: + should_create: + type: boolean + type: type: string + enum: [bugfix, feature, improvement, doc, misc, significant] rationale: type: string""" From a88ebe672dd2b925bfcbcb8faec31232c0a084c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Thu, 2 Jul 2026 23:38:27 +0800 Subject: [PATCH 10/16] fix: harden skill-eval harness and add prek hook tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Verify CLAUDE.md → AGENTS.md symlink before running; baseline arm removes both - Pin promptfoo to 0.121.17 - Fail fast on git worktree errors and clean up worktrees on partial failure - Reject SKILL_NAME + --full (baseline arm would always fail skill-used assert) - Add tests for check_eval_reminder; match guidance file basenames exactly --- dev/skill-evals/README.md | 12 +++- dev/skill-evals/eval.py | 66 ++++++++++++++++--- scripts/ci/prek/check_eval_reminder.py | 8 ++- .../tests/ci/prek/test_check_eval_reminder.py | 66 +++++++++++++++++++ 4 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 scripts/tests/ci/prek/test_check_eval_reminder.py diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 5cd1bef3e39da..0e3f937ae9a61 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -41,6 +41,11 @@ Each arm is a **git worktree** of the real repo — the agent sees actual source files (`pyproject.toml`, directory structure, etc.). The only difference between arms is which `AGENTS.md` is present. +The agent reads guidance through the `CLAUDE.md → AGENTS.md` symlink, +so the harness verifies that symlink exists (in the working tree and +on the base branch) before running and aborts if it is broken — a +regular-file CLAUDE.md would make every arm read identical guidance. + ## Prerequisites - **Node.js >=22.22.0** — check with `node --version` @@ -68,7 +73,8 @@ uv run dev/skill-evals/eval.py --repeat 3 # Add baseline arm (no AGENTS.md) to measure raw model capability: uv run dev/skill-evals/eval.py --full -# Test a skill alongside AGENTS.md: +# Test a skill alongside AGENTS.md (not combinable with --full — the +# baseline arm has no skill, so the skill-used assertion would always fail): SKILL_NAME=airflow-contribution uv run dev/skill-evals/eval.py # Use a cheaper model for fast iteration: @@ -77,8 +83,8 @@ MODEL=claude-haiku-4-5-20251001 uv run dev/skill-evals/eval.py # Disable cache (force fresh LLM calls): uv run dev/skill-evals/eval.py --no-cache -# View results in browser: -npx promptfoo@latest view +# View results in browser (use the pinned version printed by eval.py): +npx promptfoo@0.121.17 view ``` ## Adding cases diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index d0f48908bf378..34636d5f88c09 100644 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -41,7 +41,7 @@ import tempfile from pathlib import Path -PROMPTFOO_VERSION = "latest" +PROMPTFOO_VERSION = "0.121.17" REPO_ROOT = Path(__file__).resolve().parent.parent.parent SCRIPT_DIR = Path(__file__).resolve().parent @@ -87,6 +87,29 @@ def check_prerequisites() -> None: sys.exit(1) +def check_claude_md_symlink(base_branch: str) -> None: + """Verify CLAUDE.md is a symlink to AGENTS.md, in the working tree and on the base branch. + + The Claude Agent SDK reads CLAUDE.md. Swapping AGENTS.md between arms only + affects the agent while CLAUDE.md resolves to it — if CLAUDE.md ever becomes + a regular file, every arm would read identical guidance and the eval would + silently measure nothing. + """ + link = REPO_ROOT / "CLAUDE.md" + working_ok = link.is_symlink() and os.readlink(link) == "AGENTS.md" + tree_entry = run(["git", "-C", str(REPO_ROOT), "ls-tree", base_branch, "CLAUDE.md"]).stdout + target = run(["git", "-C", str(REPO_ROOT), "show", f"{base_branch}:CLAUDE.md"]).stdout + branch_ok = tree_entry.startswith("120000") and target == "AGENTS.md" + if not (working_ok and branch_ok): + print( + "Error: CLAUDE.md must be a symlink to AGENTS.md (in the working tree" + f" and on '{base_branch}'). The eval swaps AGENTS.md between arms and" + " relies on the agent reading it through that symlink.", + file=sys.stderr, + ) + sys.exit(1) + + def resolve_base_branch() -> str: result = run(["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", "main"]) if result.returncode == 0: @@ -109,12 +132,13 @@ def create_worktree( name: str, base_branch: str, agents_file: Path | None, + worktrees: list[Path], skill_name: str | None = None, skill_file: Path | None = None, ) -> Path: """Create a git worktree arm with the specified AGENTS.md and optional SKILL.md.""" wt_dir = work_dir / name - run( + result = run( [ "git", "-C", @@ -127,9 +151,21 @@ def create_worktree( base_branch, ] ) + if result.returncode != 0: + print( + f"Error: 'git worktree add' failed for arm '{name}':\n{result.stderr.strip()}", + file=sys.stderr, + ) + sys.exit(1) + # Register before mutating the checkout so cleanup covers this worktree + # even if a later step in this function fails. + worktrees.append(wt_dir) if agents_file is None: + # Baseline arm: remove all guidance. CLAUDE.md is a symlink to + # AGENTS.md — drop it too so the SDK finds no dangling link. (wt_dir / "AGENTS.md").unlink(missing_ok=True) + (wt_dir / "CLAUDE.md").unlink(missing_ok=True) else: shutil.copy2(agents_file, wt_dir / "AGENTS.md") @@ -182,7 +218,16 @@ def main() -> int: full_mode = "--full" in sys.argv promptfoo_args = [a for a in sys.argv[1:] if a != "--full"] + if full_mode and skill_name: + print( + "Error: --full cannot be combined with SKILL_NAME — the baseline arm has" + " no skill, so the 'skill-used' assertion would always fail on it.", + file=sys.stderr, + ) + return 1 + base_branch = resolve_base_branch() + check_claude_md_symlink(base_branch) # Temp dir for config and worktrees work_dir = Path(tempfile.mkdtemp()) @@ -214,23 +259,26 @@ def main() -> int: skill_changed = run(["diff", "-q", str(main_skill), str(skill_src)]).returncode != 0 need_working = agents_changed or skill_changed - # Assemble arms + # Assemble arms. Prune first to self-heal stale worktree registrations + # left behind by a previously interrupted run. print("Assembling arms (git worktrees) ...") + run(["git", "-C", str(REPO_ROOT), "worktree", "prune"]) - arm_main = create_worktree(work_dir, "main", base_branch, main_agents, skill_name, main_skill) - worktrees.append(arm_main) + arm_main = create_worktree( + work_dir, "main", base_branch, main_agents, worktrees, skill_name, main_skill + ) arm_working = None if need_working: - arm_working = create_worktree(work_dir, "working", base_branch, AGENTS_SRC, skill_name, skill_src) - worktrees.append(arm_working) + arm_working = create_worktree( + work_dir, "working", base_branch, AGENTS_SRC, worktrees, skill_name, skill_src + ) else: print(" AGENTS.md unchanged — skipping working arm") arm_baseline = None if full_mode: - arm_baseline = create_worktree(work_dir, "baseline", base_branch, None) - worktrees.append(arm_baseline) + arm_baseline = create_worktree(work_dir, "baseline", base_branch, None, worktrees) # Generate config config_lines: list[str] = [] diff --git a/scripts/ci/prek/check_eval_reminder.py b/scripts/ci/prek/check_eval_reminder.py index 1d04dadd27346..2220bbcf9aba3 100755 --- a/scripts/ci/prek/check_eval_reminder.py +++ b/scripts/ci/prek/check_eval_reminder.py @@ -37,9 +37,13 @@ def get_staged_files() -> list[str]: return result.stdout.strip().splitlines() +def find_guidance_files(staged: list[str]) -> list[str]: + """Return staged paths whose basename is exactly AGENTS.md or SKILL.md.""" + return [f for f in staged if f.rsplit("/", 1)[-1] in ("AGENTS.md", "SKILL.md")] + + def main() -> int: - staged = get_staged_files() - guidance_files = [f for f in staged if f.endswith("AGENTS.md") or f.endswith("SKILL.md")] + guidance_files = find_guidance_files(get_staged_files()) if guidance_files: changed = ", ".join(guidance_files) print( diff --git a/scripts/tests/ci/prek/test_check_eval_reminder.py b/scripts/tests/ci/prek/test_check_eval_reminder.py new file mode 100644 index 0000000000000..8e8a060847869 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_eval_reminder.py @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest +from ci.prek.check_eval_reminder import find_guidance_files, main + + +@pytest.mark.parametrize( + ("staged", "expected"), + [ + pytest.param([], [], id="empty"), + pytest.param(["AGENTS.md"], ["AGENTS.md"], id="root-agents"), + pytest.param(["providers/AGENTS.md"], ["providers/AGENTS.md"], id="nested-agents"), + pytest.param( + [".agents/skills/airflow-translations/SKILL.md"], + [".agents/skills/airflow-translations/SKILL.md"], + id="nested-skill", + ), + pytest.param(["airflow-core/src/airflow/models/dag.py", "README.md"], [], id="unrelated"), + pytest.param(["MYAGENTS.md", "docs/OLDSKILL.md"], [], id="basename-must-match-exactly"), + pytest.param( + ["AGENTS.md", "task-sdk/AGENTS.md", "dev/foo.py"], + ["AGENTS.md", "task-sdk/AGENTS.md"], + id="mixed", + ), + ], +) +def test_find_guidance_files(staged, expected): + assert find_guidance_files(staged) == expected + + +@mock.patch("ci.prek.check_eval_reminder.get_staged_files", autospec=True) +def test_main_prints_reminder_for_guidance_files(mock_get_staged_files, capsys): + mock_get_staged_files.return_value = ["AGENTS.md", "dev/foo.py"] + + assert main() == 0 + + err = capsys.readouterr().err + assert "AGENTS.md" in err + assert "dev/skill-evals/eval.py" in err + + +@mock.patch("ci.prek.check_eval_reminder.get_staged_files", autospec=True) +def test_main_is_silent_and_passes_without_guidance_files(mock_get_staged_files, capsys): + mock_get_staged_files.return_value = ["airflow-core/src/airflow/models/dag.py"] + + assert main() == 0 + + assert capsys.readouterr().err == "" From f25ee0316f7c5df3777b15a2e7c6eba91e97ff1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sat, 4 Jul 2026 09:26:01 +0900 Subject: [PATCH 11/16] feat: replace eval reminder hook with hash-based proof gate --- .pre-commit-config.yaml | 12 +- .rat-excludes | 1 + dev/skill-evals/README.md | 31 +++-- dev/skill-evals/eval.py | 16 +++ dev/skill-evals/last-eval-hash.txt | 3 + scripts/ci/prek/check_eval_hash.py | 103 ++++++++++++++ scripts/ci/prek/check_eval_reminder.py | 58 -------- scripts/tests/ci/prek/test_check_eval_hash.py | 129 ++++++++++++++++++ .../tests/ci/prek/test_check_eval_reminder.py | 66 --------- 9 files changed, 278 insertions(+), 141 deletions(-) create mode 100644 dev/skill-evals/last-eval-hash.txt create mode 100755 scripts/ci/prek/check_eval_hash.py delete mode 100755 scripts/ci/prek/check_eval_reminder.py create mode 100644 scripts/tests/ci/prek/test_check_eval_hash.py delete mode 100644 scripts/tests/ci/prek/test_check_eval_reminder.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1e253bb44b1d..bd793e74c1768 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1008,16 +1008,16 @@ repos: ^scripts/ci/prek/generate_agent_skills\.py$ pass_filenames: false additional_dependencies: ['pyyaml', 'rich>=13.6.0'] - - id: check-eval-reminder - name: Remind to run skill-eval on guidance changes - entry: ./scripts/ci/prek/check_eval_reminder.py + - id: check-eval-hash + name: Verify skill-eval ran on current AGENTS.md and cases + entry: ./scripts/ci/prek/check_eval_hash.py language: python files: > (?x) - ^(?:.*/)?AGENTS\.md$| - ^(?:.*/)?SKILL\.md$ + ^AGENTS\.md$| + ^dev/skill-evals/cases/| + ^dev/skill-evals/last-eval-hash\.txt$ pass_filenames: false - verbose: true - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml language: python diff --git a/.rat-excludes b/.rat-excludes index f0cd2910f5b27..b84b09160e658 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -158,6 +158,7 @@ PKG-INFO # checksum files .*\.md5sum +last-eval-hash.txt # Openapi files .openapi-generator-ignore diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 0e3f937ae9a61..078b59a1430cc 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -27,7 +27,7 @@ - [Usage](#usage) - [Adding cases](#adding-cases) - [How it works](#how-it-works) - - [Prek hook](#prek-hook) + - [Eval-run hash gate](#eval-run-hash-gate) @@ -117,13 +117,22 @@ Use `output.should_create` directly in assertions. 3. Runs each case against all arms in parallel. 4. Reports pass/fail diff. Worktrees cleaned up on exit. -## Prek hook - -A prek hook prints a reminder when `AGENTS.md` or `SKILL.md` is -staged for commit: - -``` -⚠ AGENTS.md modified — consider running uv run dev/skill-evals/eval.py before pushing -``` - -The hook never blocks the commit — it is informational only. +## Eval-run hash gate + +After every completed run, `eval.py` records a hash of its inputs +(`AGENTS.md` + `cases/*.yaml`) in `last-eval-hash.txt`. The +`check-eval-hash` prek hook — enforced locally and in CI — recomputes +the hash and fails when guidance changed without a re-run. Commit the +updated hash file together with your change. + +- The hash proves the eval **ran** on this exact content, not that all + cases passed (some cases fail even on `main` — that is signal, not a + defect). +- WIP commits: `SKIP=check-eval-hash git commit ...` — CI stays red + until the eval is re-run. +- Can't run the eval (no Claude subscription or API key)? Ask a + maintainer to run it and push the updated hash file to your PR branch. +- `SKILL.md` files are not covered by the gate yet — there are no skill + cases, so a hash over them would prove nothing. Extend + `compute_guidance_hash` in `scripts/ci/prek/check_eval_hash.py` when + per-skill cases land. diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index 34636d5f88c09..c94268a63f673 100644 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -46,6 +46,12 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent SCRIPT_DIR = Path(__file__).resolve().parent AGENTS_SRC = REPO_ROOT / "AGENTS.md" +CASES_DIR = SCRIPT_DIR / "cases" +HASH_FILE = SCRIPT_DIR / "last-eval-hash.txt" + +# Shared with the check-eval-hash prek hook so recorded and verified hashes can't drift. +sys.path.insert(0, str(REPO_ROOT / "scripts" / "ci" / "prek")) +from check_eval_hash import compute_guidance_hash, write_recorded_hash # noqa: E402 OUTPUT_SCHEMA = """\ output_format: @@ -229,6 +235,9 @@ def main() -> int: base_branch = resolve_base_branch() check_claude_md_symlink(base_branch) + # Hash before building arms so edits made mid-run aren't recorded as tested. + guidance_hash = compute_guidance_hash(AGENTS_SRC, CASES_DIR) + # Temp dir for config and worktrees work_dir = Path(tempfile.mkdtemp()) worktrees: list[Path] = [] @@ -336,6 +345,13 @@ def main() -> int: check=False, ) + # 0 = all passed, 100 = some assertions failed — both mean the eval + # completed and the results were seen, which is what the hash proves. + if result.returncode in (0, 100): + write_recorded_hash(guidance_hash, HASH_FILE) + print() + print(f"Recorded eval run in {HASH_FILE.relative_to(REPO_ROOT)} — commit it with your change.") + print() print(f"View results: npx promptfoo@{PROMPTFOO_VERSION} view") return result.returncode diff --git a/dev/skill-evals/last-eval-hash.txt b/dev/skill-evals/last-eval-hash.txt new file mode 100644 index 0000000000000..15c48018fc8fc --- /dev/null +++ b/dev/skill-evals/last-eval-hash.txt @@ -0,0 +1,3 @@ +# Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. +# Run `uv run dev/skill-evals/eval.py --repeat 3` to regenerate. +f959b6005d0efc11a2de0030b25b1f804788fd50118ba652ac2f7c71dfaf1d38 diff --git a/scripts/ci/prek/check_eval_hash.py b/scripts/ci/prek/check_eval_hash.py new file mode 100755 index 0000000000000..ba64b16acecb7 --- /dev/null +++ b/scripts/ci/prek/check_eval_hash.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Verify the skill-eval ran against the current guidance files. + +``dev/skill-evals/eval.py`` records a hash of its inputs in +``dev/skill-evals/last-eval-hash.txt`` after every completed run. This hook +recomputes the hash from the current file contents and fails when AGENTS.md +or the eval cases changed without a re-run. The hash proves the eval *ran* +on this exact content — not that all cases passed. + +The comparison is pure file content — no git state — so the hook behaves +identically at commit time, at push time, and in CI (``prek --all-files``). +""" + +from __future__ import annotations + +import hashlib +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +AGENTS_FILE = REPO_ROOT / "AGENTS.md" +CASES_DIR = REPO_ROOT / "dev" / "skill-evals" / "cases" +HASH_FILE = REPO_ROOT / "dev" / "skill-evals" / "last-eval-hash.txt" + +HASH_FILE_HEADER = """\ +# Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. +# Run `uv run dev/skill-evals/eval.py --repeat 3` to regenerate. +""" + + +def compute_guidance_hash(agents_file: Path, cases_dir: Path) -> str: + """Hash the skill-eval inputs: AGENTS.md plus every case file. + + Labels are hashed alongside contents so renaming a case file changes + the hash. SKILL.md files are intentionally excluded — no skill cases + exist yet; extend the inputs when per-skill cases land (runs with + ``SKILL_NAME=...``). + """ + digest = hashlib.sha256() + entries = [("AGENTS.md", agents_file)] + entries += [(f"cases/{path.name}", path) for path in sorted(cases_dir.glob("*.yaml"))] + for label, path in entries: + digest.update(label.encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def read_recorded_hash(hash_file: Path) -> str | None: + """Return the hash recorded in ``hash_file``, or None if absent.""" + if not hash_file.is_file(): + return None + for line in hash_file.read_text().splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + return stripped + return None + + +def write_recorded_hash(value: str, hash_file: Path) -> None: + """Record ``value`` with a regeneration note, like breeze's output-commands-hash.txt.""" + hash_file.write_text(f"{HASH_FILE_HEADER}{value}\n") + + +def main() -> int: + current = compute_guidance_hash(AGENTS_FILE, CASES_DIR) + recorded = read_recorded_hash(HASH_FILE) + if current == recorded: + return 0 + print( + "AGENTS.md or dev/skill-evals/cases/ changed without re-running the" + " skill-eval\n(dev/skill-evals/last-eval-hash.txt does not match).\n" + "\n" + " Run: uv run dev/skill-evals/eval.py --repeat 3\n" + " then commit the updated dev/skill-evals/last-eval-hash.txt\n" + "\n" + " WIP commit: SKIP=check-eval-hash git commit ...\n" + " Can't run the eval? Ask a maintainer to run it and push the updated\n" + " hash file to your PR branch (see dev/skill-evals/README.md).", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/prek/check_eval_reminder.py b/scripts/ci/prek/check_eval_reminder.py deleted file mode 100755 index 2220bbcf9aba3..0000000000000 --- a/scripts/ci/prek/check_eval_reminder.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Remind contributors to run skill-eval when guidance files change. - -This hook prints a warning when AGENTS.md or SKILL.md is staged for -commit. It always exits 0 — it never blocks the commit. -""" - -from __future__ import annotations - -import subprocess -import sys - - -def get_staged_files() -> list[str]: - result = subprocess.run( - ["git", "diff", "--cached", "--name-only"], - capture_output=True, - text=True, - check=False, - ) - return result.stdout.strip().splitlines() - - -def find_guidance_files(staged: list[str]) -> list[str]: - """Return staged paths whose basename is exactly AGENTS.md or SKILL.md.""" - return [f for f in staged if f.rsplit("/", 1)[-1] in ("AGENTS.md", "SKILL.md")] - - -def main() -> int: - guidance_files = find_guidance_files(get_staged_files()) - if guidance_files: - changed = ", ".join(guidance_files) - print( - f"\n\033[33m⚠ {changed} modified — " - f"consider running 'uv run dev/skill-evals/eval.py' before pushing\033[0m\n", - file=sys.stderr, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/tests/ci/prek/test_check_eval_hash.py b/scripts/tests/ci/prek/test_check_eval_hash.py new file mode 100644 index 0000000000000..a811101752175 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_eval_hash.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest +from ci.prek.check_eval_hash import ( + compute_guidance_hash, + main, + read_recorded_hash, + write_recorded_hash, +) + + +@pytest.fixture +def guidance(tmp_path): + """A tmp AGENTS.md and cases dir mirroring the eval input layout.""" + agents_file = tmp_path / "AGENTS.md" + agents_file.write_text("golden rule\n") + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "newsfragment.yaml").write_text("- description: case\n") + return agents_file, cases_dir + + +def test_compute_guidance_hash_is_deterministic(guidance): + agents_file, cases_dir = guidance + + assert compute_guidance_hash(agents_file, cases_dir) == compute_guidance_hash(agents_file, cases_dir) + + +@pytest.mark.parametrize( + "mutate", + [ + pytest.param( + lambda agents_file, cases_dir: agents_file.write_text("golden rule, revised\n"), + id="agents-content", + ), + pytest.param( + lambda agents_file, cases_dir: (cases_dir / "newsfragment.yaml").write_text( + "- description: revised case\n" + ), + id="case-content", + ), + pytest.param( + lambda agents_file, cases_dir: (cases_dir / "routing.yaml").write_text( + "- description: another case\n" + ), + id="case-added", + ), + pytest.param( + lambda agents_file, cases_dir: (cases_dir / "newsfragment.yaml").rename( + cases_dir / "renamed.yaml" + ), + id="case-renamed", + ), + ], +) +def test_compute_guidance_hash_changes_when_inputs_change(guidance, mutate): + agents_file, cases_dir = guidance + before = compute_guidance_hash(agents_file, cases_dir) + + mutate(agents_file, cases_dir) + + assert compute_guidance_hash(agents_file, cases_dir) != before + + +def test_read_recorded_hash_returns_none_when_missing(tmp_path): + assert read_recorded_hash(tmp_path / "absent.txt") is None + + +def test_write_then_read_roundtrip(tmp_path): + hash_file = tmp_path / "last-eval-hash.txt" + + write_recorded_hash("abc123", hash_file) + + assert read_recorded_hash(hash_file) == "abc123" + assert hash_file.read_text().startswith("# Generated by dev/skill-evals/eval.py") + assert hash_file.read_text().endswith("abc123\n") + + +def test_main_passes_when_hash_matches(guidance, tmp_path, capsys): + agents_file, cases_dir = guidance + hash_file = tmp_path / "last-eval-hash.txt" + write_recorded_hash(compute_guidance_hash(agents_file, cases_dir), hash_file) + + with ( + mock.patch("ci.prek.check_eval_hash.AGENTS_FILE", agents_file), + mock.patch("ci.prek.check_eval_hash.CASES_DIR", cases_dir), + mock.patch("ci.prek.check_eval_hash.HASH_FILE", hash_file), + ): + assert main() == 0 + + assert capsys.readouterr().err == "" + + +@pytest.mark.parametrize("record_stale_hash", [True, False], ids=["stale-hash", "missing-file"]) +def test_main_fails_on_mismatch_or_missing_record(guidance, tmp_path, capsys, record_stale_hash): + agents_file, cases_dir = guidance + hash_file = tmp_path / "last-eval-hash.txt" + if record_stale_hash: + write_recorded_hash(compute_guidance_hash(agents_file, cases_dir), hash_file) + agents_file.write_text("changed after the recorded run\n") + + with ( + mock.patch("ci.prek.check_eval_hash.AGENTS_FILE", agents_file), + mock.patch("ci.prek.check_eval_hash.CASES_DIR", cases_dir), + mock.patch("ci.prek.check_eval_hash.HASH_FILE", hash_file), + ): + assert main() == 1 + + err = capsys.readouterr().err + assert "uv run dev/skill-evals/eval.py" in err + assert "SKIP=check-eval-hash" in err diff --git a/scripts/tests/ci/prek/test_check_eval_reminder.py b/scripts/tests/ci/prek/test_check_eval_reminder.py deleted file mode 100644 index 8e8a060847869..0000000000000 --- a/scripts/tests/ci/prek/test_check_eval_reminder.py +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -from unittest import mock - -import pytest -from ci.prek.check_eval_reminder import find_guidance_files, main - - -@pytest.mark.parametrize( - ("staged", "expected"), - [ - pytest.param([], [], id="empty"), - pytest.param(["AGENTS.md"], ["AGENTS.md"], id="root-agents"), - pytest.param(["providers/AGENTS.md"], ["providers/AGENTS.md"], id="nested-agents"), - pytest.param( - [".agents/skills/airflow-translations/SKILL.md"], - [".agents/skills/airflow-translations/SKILL.md"], - id="nested-skill", - ), - pytest.param(["airflow-core/src/airflow/models/dag.py", "README.md"], [], id="unrelated"), - pytest.param(["MYAGENTS.md", "docs/OLDSKILL.md"], [], id="basename-must-match-exactly"), - pytest.param( - ["AGENTS.md", "task-sdk/AGENTS.md", "dev/foo.py"], - ["AGENTS.md", "task-sdk/AGENTS.md"], - id="mixed", - ), - ], -) -def test_find_guidance_files(staged, expected): - assert find_guidance_files(staged) == expected - - -@mock.patch("ci.prek.check_eval_reminder.get_staged_files", autospec=True) -def test_main_prints_reminder_for_guidance_files(mock_get_staged_files, capsys): - mock_get_staged_files.return_value = ["AGENTS.md", "dev/foo.py"] - - assert main() == 0 - - err = capsys.readouterr().err - assert "AGENTS.md" in err - assert "dev/skill-evals/eval.py" in err - - -@mock.patch("ci.prek.check_eval_reminder.get_staged_files", autospec=True) -def test_main_is_silent_and_passes_without_guidance_files(mock_get_staged_files, capsys): - mock_get_staged_files.return_value = ["airflow-core/src/airflow/models/dag.py"] - - assert main() == 0 - - assert capsys.readouterr().err == "" From f065bc9bb89cb0248de6076959fcecc0c5c46f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sat, 4 Jul 2026 10:28:51 +0900 Subject: [PATCH 12/16] feat: run skill-eval via prek-managed node env, guard partial runs --- .pre-commit-config.yaml | 9 ++++++++ dev/skill-evals/README.md | 29 +++++++++++++++-------- dev/skill-evals/eval.py | 37 +++++++++++++++++++++++------- dev/skill-evals/last-eval-hash.txt | 2 +- scripts/ci/prek/check_eval_hash.py | 5 ++-- 5 files changed, 61 insertions(+), 21 deletions(-) mode change 100644 => 100755 dev/skill-evals/eval.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd793e74c1768..c3aa5fc83c170 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1008,6 +1008,15 @@ repos: ^scripts/ci/prek/generate_agent_skills\.py$ pass_filenames: false additional_dependencies: ['pyyaml', 'rich>=13.6.0'] + - id: run-skill-eval + name: Run skill-eval against AGENTS.md and cases + entry: ./dev/skill-evals/eval.py + language: node + language_version: '22.22.0' + additional_dependencies: ['promptfoo@0.121.17', '@anthropic-ai/claude-agent-sdk@0.3.185'] + stages: ['manual'] + pass_filenames: false + require_serial: true - id: check-eval-hash name: Verify skill-eval ran on current AGENTS.md and cases entry: ./scripts/ci/prek/check_eval_hash.py diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 078b59a1430cc..3f93a0ddbcfa9 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -23,7 +23,6 @@ - [Skill-Eval Harness](#skill-eval-harness) - [Prerequisites](#prerequisites) - - [One-time setup](#one-time-setup) - [Usage](#usage) - [Adding cases](#adding-cases) - [How it works](#how-it-works) @@ -48,23 +47,23 @@ regular-file CLAUDE.md would make every arm read identical guidance. ## Prerequisites -- **Node.js >=22.22.0** — check with `node --version` - **Authentication** (one of): - Claude Code session (`claude /login`) — Pro/Max subscription - `ANTHROPIC_API_KEY` environment variable — API credits -## One-time setup - -Install the Claude Agent SDK (promptfoo needs it at runtime): - -```bash -mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk -``` +That's it for the prek path below — prek provisions Node, promptfoo, and +the Claude Agent SDK automatically. The direct path needs extra setup +(see below). ## Usage ```bash -# Check for regressions after editing AGENTS.md: +# Zero-setup: run the eval through the prek-provisioned environment +# (single pass over all cases — satisfies the hash gate): +prek run run-skill-eval --hook-stage manual + +# Direct invocation — accepts promptfoo flags; needs Node.js >=22.22.0 +# and the one-time SDK setup below: uv run dev/skill-evals/eval.py # Repeat each case to reduce nondeterminism: @@ -87,6 +86,16 @@ uv run dev/skill-evals/eval.py --no-cache npx promptfoo@0.121.17 view ``` +One-time setup for the **direct path only** — install the Claude Agent +SDK where promptfoo can resolve it: + +```bash +mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk +``` + +A run with `--filter*` flags covers only a subset of cases, so it does +not update the hash file. + ## Adding cases Cases live in `cases/*.yaml`. Add entries to an existing file or diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py old mode 100644 new mode 100755 index c94268a63f673..9adc97d0d239e --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -74,7 +74,19 @@ def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess[str]: return subprocess.run(cmd, capture_output=True, text=True, check=False, **kwargs) -def check_prerequisites() -> None: +def resolve_promptfoo_command() -> list[str]: + """Use promptfoo from PATH if it matches the pinned version (the prek run-skill-eval env), else npx.""" + if shutil.which("promptfoo"): + version = run(["promptfoo", "--version"]).stdout.strip() + if version == PROMPTFOO_VERSION: + return ["promptfoo"] + return ["npx", f"promptfoo@{PROMPTFOO_VERSION}"] + + +def check_prerequisites(promptfoo_cmd: list[str]) -> None: + if promptfoo_cmd[0] == "promptfoo": + return # prek env provides node, promptfoo, and the SDK + if not shutil.which("node"): print("Error: Node.js not found. Install Node.js >=22.22.0", file=sys.stderr) sys.exit(1) @@ -209,7 +221,8 @@ def add_provider( def main() -> int: - check_prerequisites() + promptfoo_cmd = resolve_promptfoo_command() + check_prerequisites(promptfoo_cmd) model = os.environ.get("MODEL", "claude-sonnet-4-6") skill_name = os.environ.get("SKILL_NAME") @@ -243,9 +256,11 @@ def main() -> int: worktrees: list[Path] = [] try: - # Symlink node_modules for promptfoo SDK resolution - sdk_modules = Path.home() / ".promptfoo-sdk" / "node_modules" - (work_dir / "node_modules").symlink_to(sdk_modules) + # Symlink node_modules for promptfoo SDK resolution (npx path only — + # the prek env bundles the SDK inside promptfoo's own node_modules) + if promptfoo_cmd[0] != "promptfoo": + sdk_modules = Path.home() / ".promptfoo-sdk" / "node_modules" + (work_dir / "node_modules").symlink_to(sdk_modules) # Extract main-branch AGENTS.md main_agents = work_dir / "main-agents.md" @@ -341,19 +356,25 @@ def main() -> int: # Run promptfoo result = subprocess.run( - ["npx", f"promptfoo@{PROMPTFOO_VERSION}", "eval", "-c", str(config_path), *promptfoo_args], + [*promptfoo_cmd, "eval", "-c", str(config_path), *promptfoo_args], check=False, ) # 0 = all passed, 100 = some assertions failed — both mean the eval # completed and the results were seen, which is what the hash proves. - if result.returncode in (0, 100): + # A filtered run covers only a subset of cases, so it proves nothing + # about the full set — don't record it. + partial_run = any(arg.startswith("--filter") for arg in promptfoo_args) + if result.returncode in (0, 100) and not partial_run: write_recorded_hash(guidance_hash, HASH_FILE) print() print(f"Recorded eval run in {HASH_FILE.relative_to(REPO_ROOT)} — commit it with your change.") + elif partial_run: + print() + print("Partial run (--filter*) — hash not recorded; run the full case set to update it.") print() - print(f"View results: npx promptfoo@{PROMPTFOO_VERSION} view") + print(f"View results: {' '.join(promptfoo_cmd)} view") return result.returncode finally: diff --git a/dev/skill-evals/last-eval-hash.txt b/dev/skill-evals/last-eval-hash.txt index 15c48018fc8fc..3f7457e754f87 100644 --- a/dev/skill-evals/last-eval-hash.txt +++ b/dev/skill-evals/last-eval-hash.txt @@ -1,3 +1,3 @@ # Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. -# Run `uv run dev/skill-evals/eval.py --repeat 3` to regenerate. +# Run `prek run run-skill-eval --hook-stage manual` to regenerate. f959b6005d0efc11a2de0030b25b1f804788fd50118ba652ac2f7c71dfaf1d38 diff --git a/scripts/ci/prek/check_eval_hash.py b/scripts/ci/prek/check_eval_hash.py index ba64b16acecb7..ede98b158c4de 100755 --- a/scripts/ci/prek/check_eval_hash.py +++ b/scripts/ci/prek/check_eval_hash.py @@ -40,7 +40,7 @@ HASH_FILE_HEADER = """\ # Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. -# Run `uv run dev/skill-evals/eval.py --repeat 3` to regenerate. +# Run `prek run run-skill-eval --hook-stage manual` to regenerate. """ @@ -88,7 +88,8 @@ def main() -> int: "AGENTS.md or dev/skill-evals/cases/ changed without re-running the" " skill-eval\n(dev/skill-evals/last-eval-hash.txt does not match).\n" "\n" - " Run: uv run dev/skill-evals/eval.py --repeat 3\n" + " Run: prek run run-skill-eval --hook-stage manual\n" + " (or: uv run dev/skill-evals/eval.py)\n" " then commit the updated dev/skill-evals/last-eval-hash.txt\n" "\n" " WIP commit: SKIP=check-eval-hash git commit ...\n" From 5e53db04a980952ae01c711fa6b3f3e8f8b3c405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sat, 4 Jul 2026 22:01:54 +0900 Subject: [PATCH 13/16] feat: follow the files/ output convention for eval results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - each run exports a JSON report to files/skill-evals/ - promptfoo db/cache and the SDK install move from the home directory to .build/ (tool state, not output) — nothing lands in /Users/lizhechen anymore - run-skill-eval only triggers on guidance changes during bulk manual runs; documented invocation gains --all-files - README: cleanup section, stage-before-prek-run reminder --- .pre-commit-config.yaml | 4 ++++ dev/skill-evals/README.md | 26 +++++++++++++++++++++----- dev/skill-evals/eval.py | 27 +++++++++++++++++++-------- dev/skill-evals/last-eval-hash.txt | 2 +- scripts/ci/prek/check_eval_hash.py | 4 ++-- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3aa5fc83c170..eb57ee1b7ce0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1015,6 +1015,10 @@ repos: language_version: '22.22.0' additional_dependencies: ['promptfoo@0.121.17', '@anthropic-ai/claude-agent-sdk@0.3.185'] stages: ['manual'] + files: > + (?x) + ^AGENTS\.md$| + ^dev/skill-evals/ pass_filenames: false require_serial: true - id: check-eval-hash diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 3f93a0ddbcfa9..440259df98c4c 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -24,6 +24,7 @@ - [Skill-Eval Harness](#skill-eval-harness) - [Prerequisites](#prerequisites) - [Usage](#usage) + - [Cleanup](#cleanup) - [Adding cases](#adding-cases) - [How it works](#how-it-works) - [Eval-run hash gate](#eval-run-hash-gate) @@ -59,8 +60,9 @@ the Claude Agent SDK automatically. The direct path needs extra setup ```bash # Zero-setup: run the eval through the prek-provisioned environment -# (single pass over all cases — satisfies the hash gate): -prek run run-skill-eval --hook-stage manual +# (single pass over all cases — satisfies the hash gate). +# Stage your changes first — prek stashes unstaged edits: +prek run run-skill-eval --hook-stage manual --all-files # Direct invocation — accepts promptfoo flags; needs Node.js >=22.22.0 # and the one-time SDK setup below: @@ -82,20 +84,34 @@ MODEL=claude-haiku-4-5-20251001 uv run dev/skill-evals/eval.py # Disable cache (force fresh LLM calls): uv run dev/skill-evals/eval.py --no-cache -# View results in browser (use the pinned version printed by eval.py): -npx promptfoo@0.121.17 view +# View results in browser (state lives under .build/promptfoo): +PROMPTFOO_CONFIG_DIR=.build/promptfoo npx promptfoo@0.121.17 view ``` One-time setup for the **direct path only** — install the Claude Agent SDK where promptfoo can resolve it: ```bash -mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk +mkdir -p .build/promptfoo-sdk && cd .build/promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk ``` A run with `--filter*` flags covers only a subset of cases, so it does not update the hash file. +Each run also writes a JSON report to `files/skill-evals/results.json` +(per the `files/` output convention) — handy for pasting results into +a PR. + +## Cleanup + +Everything the harness stores lives inside the repo — nothing is left +in your home directory: + +```bash +rm -rf .build/promptfoo .build/promptfoo-sdk # eval history, cache, SDK +prek clean # prek-provisioned node envs +``` + ## Adding cases Cases live in `cases/*.yaml`. Add entries to an existing file or diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index 9adc97d0d239e..86520adba4df9 100755 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -49,6 +49,12 @@ CASES_DIR = SCRIPT_DIR / "cases" HASH_FILE = SCRIPT_DIR / "last-eval-hash.txt" +# Tool state lives under .build (repo-scoped, established cleanup culture); +# per-run reports go to files/ per the AGENTS.md output convention. +PROMPTFOO_STATE_DIR = REPO_ROOT / ".build" / "promptfoo" +SDK_DIR = REPO_ROOT / ".build" / "promptfoo-sdk" +RESULTS_FILE = REPO_ROOT / "files" / "skill-evals" / "results.json" + # Shared with the check-eval-hash prek hook so recorded and verified hashes can't drift. sys.path.insert(0, str(REPO_ROOT / "scripts" / "ci" / "prek")) from check_eval_hash import compute_guidance_hash, write_recorded_hash # noqa: E402 @@ -94,11 +100,10 @@ def check_prerequisites(promptfoo_cmd: list[str]) -> None: print("Error: npx not found.", file=sys.stderr) sys.exit(1) - sdk_dir = Path.home() / ".promptfoo-sdk" / "node_modules" / "@anthropic-ai" / "claude-agent-sdk" - if not sdk_dir.is_dir(): + if not (SDK_DIR / "node_modules" / "@anthropic-ai" / "claude-agent-sdk").is_dir(): print("Error: Claude Agent SDK not found. Run:", file=sys.stderr) print( - " mkdir -p ~/.promptfoo-sdk && cd ~/.promptfoo-sdk" + f" mkdir -p {SDK_DIR} && cd {SDK_DIR}" " && npm init -y && npm install @anthropic-ai/claude-agent-sdk", file=sys.stderr, ) @@ -259,8 +264,7 @@ def main() -> int: # Symlink node_modules for promptfoo SDK resolution (npx path only — # the prek env bundles the SDK inside promptfoo's own node_modules) if promptfoo_cmd[0] != "promptfoo": - sdk_modules = Path.home() / ".promptfoo-sdk" / "node_modules" - (work_dir / "node_modules").symlink_to(sdk_modules) + (work_dir / "node_modules").symlink_to(SDK_DIR / "node_modules") # Extract main-branch AGENTS.md main_agents = work_dir / "main-agents.md" @@ -354,10 +358,13 @@ def main() -> int: print(f" SKILL.md — {'modified' if skill_changed else 'unchanged'} ({skill_name})") print() - # Run promptfoo + # Run promptfoo — state under .build, per-run report under files/ + PROMPTFOO_STATE_DIR.mkdir(parents=True, exist_ok=True) + RESULTS_FILE.parent.mkdir(parents=True, exist_ok=True) result = subprocess.run( - [*promptfoo_cmd, "eval", "-c", str(config_path), *promptfoo_args], + [*promptfoo_cmd, "eval", "-c", str(config_path), "--output", str(RESULTS_FILE), *promptfoo_args], check=False, + env={**os.environ, "PROMPTFOO_CONFIG_DIR": str(PROMPTFOO_STATE_DIR)}, ) # 0 = all passed, 100 = some assertions failed — both mean the eval @@ -374,7 +381,11 @@ def main() -> int: print("Partial run (--filter*) — hash not recorded; run the full case set to update it.") print() - print(f"View results: {' '.join(promptfoo_cmd)} view") + print(f"Results report: {RESULTS_FILE.relative_to(REPO_ROOT)}") + print( + f"View results: PROMPTFOO_CONFIG_DIR={PROMPTFOO_STATE_DIR.relative_to(REPO_ROOT)}" + f" {' '.join(promptfoo_cmd)} view" + ) return result.returncode finally: diff --git a/dev/skill-evals/last-eval-hash.txt b/dev/skill-evals/last-eval-hash.txt index 3f7457e754f87..6d6a5961c51c7 100644 --- a/dev/skill-evals/last-eval-hash.txt +++ b/dev/skill-evals/last-eval-hash.txt @@ -1,3 +1,3 @@ # Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. -# Run `prek run run-skill-eval --hook-stage manual` to regenerate. +# Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate. f959b6005d0efc11a2de0030b25b1f804788fd50118ba652ac2f7c71dfaf1d38 diff --git a/scripts/ci/prek/check_eval_hash.py b/scripts/ci/prek/check_eval_hash.py index ede98b158c4de..74d2b3c40c9e2 100755 --- a/scripts/ci/prek/check_eval_hash.py +++ b/scripts/ci/prek/check_eval_hash.py @@ -40,7 +40,7 @@ HASH_FILE_HEADER = """\ # Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. -# Run `prek run run-skill-eval --hook-stage manual` to regenerate. +# Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate. """ @@ -88,7 +88,7 @@ def main() -> int: "AGENTS.md or dev/skill-evals/cases/ changed without re-running the" " skill-eval\n(dev/skill-evals/last-eval-hash.txt does not match).\n" "\n" - " Run: prek run run-skill-eval --hook-stage manual\n" + " Run: prek run run-skill-eval --hook-stage manual --all-files\n" " (or: uv run dev/skill-evals/eval.py)\n" " then commit the updated dev/skill-evals/last-eval-hash.txt\n" "\n" From aa223bcda3ad93c12e350b21380d43815f6ac5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sat, 4 Jul 2026 22:32:37 +0900 Subject: [PATCH 14/16] refactor: build promptfoo config as a dict, serialize to JSON --- .pre-commit-config.yaml | 1 + dev/skill-evals/eval.py | 110 ++++++++++++++++------------------------ 2 files changed, 45 insertions(+), 66 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb57ee1b7ce0b..2386a95042019 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1021,6 +1021,7 @@ repos: ^dev/skill-evals/ pass_filenames: false require_serial: true + verbose: true - id: check-eval-hash name: Verify skill-eval ran on current AGENTS.md and cases entry: ./scripts/ci/prek/check_eval_hash.py diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index 86520adba4df9..cbc7b18ac1eb4 100755 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -34,6 +34,7 @@ from __future__ import annotations +import json import os import shutil import subprocess @@ -59,21 +60,22 @@ sys.path.insert(0, str(REPO_ROOT / "scripts" / "ci" / "prek")) from check_eval_hash import compute_guidance_hash, write_recorded_hash # noqa: E402 -OUTPUT_SCHEMA = """\ - output_format: - type: json_schema - schema: - type: object - required: [should_create, rationale] - additionalProperties: false - properties: - should_create: - type: boolean - type: - type: string - enum: [bugfix, feature, improvement, doc, misc, significant] - rationale: - type: string""" +OUTPUT_FORMAT = { + "type": "json_schema", + "schema": { + "type": "object", + "required": ["should_create", "rationale"], + "additionalProperties": False, + "properties": { + "should_create": {"type": "boolean"}, + "type": { + "type": "string", + "enum": ["bugfix", "feature", "improvement", "doc", "misc", "significant"], + }, + "rationale": {"type": "string"}, + }, + }, +} def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess[str]: @@ -204,25 +206,18 @@ def remove_worktree(wt_dir: Path) -> None: run(["git", "-C", str(REPO_ROOT), "worktree", "remove", "--force", str(wt_dir)]) -def add_provider( - config_lines: list[str], - label: str, - working_dir: Path, - model: str, - skill_name: str | None = None, -) -> None: - config_lines.append(" - id: anthropic:claude-agent-sdk") - config_lines.append(f" label: {label}") - config_lines.append(" config:") - config_lines.append(f" model: {model}") - config_lines.append(" apiKeyRequired: false") - config_lines.append(" setting_sources: ['project']") - config_lines.append(" append_allowed_tools: ['Read', 'Grep', 'Glob']") - config_lines.append(f" working_dir: {working_dir}") +def build_provider(label: str, working_dir: Path, model: str, skill_name: str | None = None) -> dict: + config: dict = { + "model": model, + "apiKeyRequired": False, + "setting_sources": ["project"], + "append_allowed_tools": ["Read", "Grep", "Glob"], + "working_dir": str(working_dir), + "output_format": OUTPUT_FORMAT, + } if skill_name: - config_lines.append(f" skills: ['{skill_name}']") - config_lines.append(OUTPUT_SCHEMA) - config_lines.append("") + config["skills"] = [skill_name] + return {"id": "anthropic:claude-agent-sdk", "label": label, "config": config} def main() -> int: @@ -308,48 +303,31 @@ def main() -> int: if full_mode: arm_baseline = create_worktree(work_dir, "baseline", base_branch, None, worktrees) - # Generate config - config_lines: list[str] = [] - config_lines.append("prompts:") - config_lines.append(" - '{{request}}'") - config_lines.append("") - config_lines.append("providers:") - - arm_count = 0 - skill_for_provider = skill_name if skill_name else None - - add_provider(config_lines, "main", arm_main, model, skill_for_provider) - arm_count += 1 - + # Generate config (JSON — valid promptfoo config, keeps the script stdlib-only) + providers = [build_provider("main", arm_main, model, skill_name)] if arm_working: - add_provider(config_lines, "working", arm_working, model, skill_for_provider) - arm_count += 1 - + providers.append(build_provider("working", arm_working, model, skill_name)) if full_mode and arm_baseline: - add_provider(config_lines, "baseline", arm_baseline, model) - arm_count += 1 + providers.append(build_provider("baseline", arm_baseline, model)) - # Default test config - config_lines.append("defaultTest:") - config_lines.append(" options:") - config_lines.append(" disableVarExpansion: true") + default_test: dict = {"options": {"disableVarExpansion": True}} if skill_name: - config_lines.append(" assert:") - config_lines.append(" - type: skill-used") - config_lines.append(f" value: {skill_name}") - config_lines.append("") - - # Cases - config_lines.append(f"tests: file://{SCRIPT_DIR}/cases/*.yaml") + default_test["assert"] = [{"type": "skill-used", "value": skill_name}] - config_path = work_dir / "promptfooconfig.yaml" - config_path.write_text("\n".join(config_lines) + "\n") + config = { + "prompts": ["{{request}}"], + "providers": providers, + "defaultTest": default_test, + "tests": f"file://{SCRIPT_DIR}/cases/*.yaml", + } + config_path = work_dir / "promptfooconfig.json" + config_path.write_text(json.dumps(config, indent=2)) # Report if skill_name: - print(f"Mode: {arm_count} arms, skill '{skill_name}', model: {model}") + print(f"Mode: {len(providers)} arms, skill '{skill_name}', model: {model}") else: - print(f"Mode: {arm_count} arms, AGENTS.md only, model: {model}") + print(f"Mode: {len(providers)} arms, AGENTS.md only, model: {model}") print() print(f"Changes detected (vs {base_branch}):") From fb2e69b27ca1c237d7f2807125050e6c014bfcea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sat, 4 Jul 2026 23:51:37 +0900 Subject: [PATCH 15/16] refactor: Make prek the only entry point for the skill-eval Direct npx invocation needed a manually installed SDK (promptfoo resolves it from the config directory); the prek env provides it. eval.py now points at when promptfoo is absent. Promptfoo flags remain wireable via hook-variant entry args. --- dev/skill-evals/README.md | 42 +++---- dev/skill-evals/eval.py | 105 ++++++++++-------- scripts/ci/prek/check_eval_hash.py | 1 - scripts/tests/ci/prek/test_check_eval_hash.py | 2 +- 4 files changed, 76 insertions(+), 74 deletions(-) diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 440259df98c4c..18843322675dd 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -52,48 +52,36 @@ regular-file CLAUDE.md would make every arm read identical guidance. - Claude Code session (`claude /login`) — Pro/Max subscription - `ANTHROPIC_API_KEY` environment variable — API credits -That's it for the prek path below — prek provisions Node, promptfoo, and -the Claude Agent SDK automatically. The direct path needs extra setup -(see below). +That's it — prek provisions Node, promptfoo, and the Claude Agent SDK +automatically. ## Usage ```bash -# Zero-setup: run the eval through the prek-provisioned environment -# (single pass over all cases — satisfies the hash gate). +# Run the eval (single pass over all cases — satisfies the hash gate). # Stage your changes first — prek stashes unstaged edits: prek run run-skill-eval --hook-stage manual --all-files -# Direct invocation — accepts promptfoo flags; needs Node.js >=22.22.0 -# and the one-time SDK setup below: -uv run dev/skill-evals/eval.py - # Repeat each case to reduce nondeterminism: -uv run dev/skill-evals/eval.py --repeat 3 - -# Add baseline arm (no AGENTS.md) to measure raw model capability: -uv run dev/skill-evals/eval.py --full +EVAL_REPEAT=3 prek run run-skill-eval --hook-stage manual --all-files -# Test a skill alongside AGENTS.md (not combinable with --full — the -# baseline arm has no skill, so the skill-used assertion would always fail): -SKILL_NAME=airflow-contribution uv run dev/skill-evals/eval.py +# Add a baseline arm (no AGENTS.md) to measure raw model capability: +EVAL_FULL=1 prek run run-skill-eval --hook-stage manual --all-files # Use a cheaper model for fast iteration: -MODEL=claude-haiku-4-5-20251001 uv run dev/skill-evals/eval.py +MODEL=claude-haiku-4-5-20251001 prek run run-skill-eval --hook-stage manual --all-files -# Disable cache (force fresh LLM calls): -uv run dev/skill-evals/eval.py --no-cache +# Test a skill alongside AGENTS.md (not combinable with EVAL_FULL — the +# baseline arm has no skill, so the skill-used assertion would always fail): +SKILL_NAME=airflow-contribution prek run run-skill-eval --hook-stage manual --all-files # View results in browser (state lives under .build/promptfoo): PROMPTFOO_CONFIG_DIR=.build/promptfoo npx promptfoo@0.121.17 view ``` -One-time setup for the **direct path only** — install the Claude Agent -SDK where promptfoo can resolve it: - -```bash -mkdir -p .build/promptfoo-sdk && cd .build/promptfoo-sdk && npm init -y && npm install @anthropic-ai/claude-agent-sdk -``` +Other promptfoo flags (`--filter*`, `--no-cache`) are argv-only — +`prek run` can't forward arguments, so wire them as fixed entry args +on a hook variant if a preset is needed. A run with `--filter*` flags covers only a subset of cases, so it does not update the hash file. @@ -108,8 +96,8 @@ Everything the harness stores lives inside the repo — nothing is left in your home directory: ```bash -rm -rf .build/promptfoo .build/promptfoo-sdk # eval history, cache, SDK -prek clean # prek-provisioned node envs +rm -rf .build/promptfoo # eval history and cache +prek clean # prek-provisioned node envs ``` ## Adding cases diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index cbc7b18ac1eb4..8505f29374d78 100755 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -25,9 +25,11 @@ real repo — the agent sees actual source files. Usage: - uv run dev/skill-evals/eval.py Test AGENTS.md changes - uv run dev/skill-evals/eval.py --full Add baseline arm (no AGENTS.md) - uv run dev/skill-evals/eval.py --repeat 3 Reduce nondeterminism + prek run run-skill-eval --hook-stage manual --all-files + +Env knobs: MODEL, SKILL_NAME, EVAL_REPEAT, EVAL_FULL (baseline arm). +Promptfoo flags like --filter* are argv-only — wire them as fixed entry +args on a hook variant when needed. Authentication: Claude Code session (claude /login) or ANTHROPIC_API_KEY. """ @@ -53,7 +55,6 @@ # Tool state lives under .build (repo-scoped, established cleanup culture); # per-run reports go to files/ per the AGENTS.md output convention. PROMPTFOO_STATE_DIR = REPO_ROOT / ".build" / "promptfoo" -SDK_DIR = REPO_ROOT / ".build" / "promptfoo-sdk" RESULTS_FILE = REPO_ROOT / "files" / "skill-evals" / "results.json" # Shared with the check-eval-hash prek hook so recorded and verified hashes can't drift. @@ -82,34 +83,30 @@ def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess[str]: return subprocess.run(cmd, capture_output=True, text=True, check=False, **kwargs) -def resolve_promptfoo_command() -> list[str]: - """Use promptfoo from PATH if it matches the pinned version (the prek run-skill-eval env), else npx.""" - if shutil.which("promptfoo"): +def find_sdk_modules() -> Path: + """Locate the node_modules dir (in the prek env) that contains the Claude Agent SDK. + + promptfoo resolves the SDK from the eval config's directory, not from its + own install tree — the caller symlinks this dir next to the config. + """ + promptfoo_bin = shutil.which("promptfoo") + if promptfoo_bin: version = run(["promptfoo", "--version"]).stdout.strip() if version == PROMPTFOO_VERSION: - return ["promptfoo"] - return ["npx", f"promptfoo@{PROMPTFOO_VERSION}"] - - -def check_prerequisites(promptfoo_cmd: list[str]) -> None: - if promptfoo_cmd[0] == "promptfoo": - return # prek env provides node, promptfoo, and the SDK - - if not shutil.which("node"): - print("Error: Node.js not found. Install Node.js >=22.22.0", file=sys.stderr) - sys.exit(1) - if not shutil.which("npx"): - print("Error: npx not found.", file=sys.stderr) - sys.exit(1) - - if not (SDK_DIR / "node_modules" / "@anthropic-ai" / "claude-agent-sdk").is_dir(): - print("Error: Claude Agent SDK not found. Run:", file=sys.stderr) - print( - f" mkdir -p {SDK_DIR} && cd {SDK_DIR}" - " && npm init -y && npm install @anthropic-ai/claude-agent-sdk", - file=sys.stderr, - ) - sys.exit(1) + pf_pkg = Path(promptfoo_bin).resolve() + while pf_pkg.name != "promptfoo" or not (pf_pkg / "package.json").is_file(): + if pf_pkg.parent == pf_pkg: + break + pf_pkg = pf_pkg.parent + for candidate in (pf_pkg / "node_modules", pf_pkg.parent): + if (candidate / "@anthropic-ai" / "claude-agent-sdk").is_dir(): + return candidate + print( + "Error: promptfoo with the Claude Agent SDK not found. Run the eval via:\n" + " prek run run-skill-eval --hook-stage manual --all-files", + file=sys.stderr, + ) + sys.exit(1) def check_claude_md_symlink(base_branch: str) -> None: @@ -220,9 +217,17 @@ def build_provider(label: str, working_dir: Path, model: str, skill_name: str | return {"id": "anthropic:claude-agent-sdk", "label": label, "config": config} +def count_provider_errors(results_file: Path) -> int: + """Count results whose provider call errored (as opposed to failing an assertion).""" + try: + results = json.loads(results_file.read_text())["results"]["results"] + except (OSError, KeyError, ValueError): + return 0 + return sum(1 for r in results if (r.get("response") or {}).get("error")) + + def main() -> int: - promptfoo_cmd = resolve_promptfoo_command() - check_prerequisites(promptfoo_cmd) + sdk_modules = find_sdk_modules() model = os.environ.get("MODEL", "claude-sonnet-4-6") skill_name = os.environ.get("SKILL_NAME") @@ -233,14 +238,22 @@ def main() -> int: print(f"Error: {skill_src} not found", file=sys.stderr) return 1 - # Parse flags - full_mode = "--full" in sys.argv + # Parse flags — argv (hook entry args) plus single-value env knobs, + # since `prek run` can't forward arguments. No filter knob on purpose: + # a lingering export must not be able to change what the proof claims. + full_mode = "--full" in sys.argv or os.environ.get("EVAL_FULL", "").lower() in ("1", "true") promptfoo_args = [a for a in sys.argv[1:] if a != "--full"] + repeat = os.environ.get("EVAL_REPEAT") + if repeat: + if not repeat.isdigit() or int(repeat) < 1: + print(f"Error: EVAL_REPEAT must be a positive integer, got {repeat!r}", file=sys.stderr) + return 1 + promptfoo_args += ["--repeat", repeat] if full_mode and skill_name: print( - "Error: --full cannot be combined with SKILL_NAME — the baseline arm has" - " no skill, so the 'skill-used' assertion would always fail on it.", + "Error: EVAL_FULL/--full cannot be combined with SKILL_NAME — the baseline" + " arm has no skill, so the 'skill-used' assertion would always fail on it.", file=sys.stderr, ) return 1 @@ -256,10 +269,8 @@ def main() -> int: worktrees: list[Path] = [] try: - # Symlink node_modules for promptfoo SDK resolution (npx path only — - # the prek env bundles the SDK inside promptfoo's own node_modules) - if promptfoo_cmd[0] != "promptfoo": - (work_dir / "node_modules").symlink_to(SDK_DIR / "node_modules") + # promptfoo resolves the Claude Agent SDK from the config directory + (work_dir / "node_modules").symlink_to(sdk_modules) # Extract main-branch AGENTS.md main_agents = work_dir / "main-agents.md" @@ -340,29 +351,33 @@ def main() -> int: PROMPTFOO_STATE_DIR.mkdir(parents=True, exist_ok=True) RESULTS_FILE.parent.mkdir(parents=True, exist_ok=True) result = subprocess.run( - [*promptfoo_cmd, "eval", "-c", str(config_path), "--output", str(RESULTS_FILE), *promptfoo_args], + ["promptfoo", "eval", "-c", str(config_path), "--output", str(RESULTS_FILE), *promptfoo_args], check=False, env={**os.environ, "PROMPTFOO_CONFIG_DIR": str(PROMPTFOO_STATE_DIR)}, ) # 0 = all passed, 100 = some assertions failed — both mean the eval # completed and the results were seen, which is what the hash proves. - # A filtered run covers only a subset of cases, so it proves nothing - # about the full set — don't record it. + # Don't record partial runs (--filter* covers a subset of cases) or + # runs with provider errors (nothing was actually evaluated). partial_run = any(arg.startswith("--filter") for arg in promptfoo_args) - if result.returncode in (0, 100) and not partial_run: + provider_errors = count_provider_errors(RESULTS_FILE) + if result.returncode in (0, 100) and not partial_run and not provider_errors: write_recorded_hash(guidance_hash, HASH_FILE) print() print(f"Recorded eval run in {HASH_FILE.relative_to(REPO_ROOT)} — commit it with your change.") elif partial_run: print() print("Partial run (--filter*) — hash not recorded; run the full case set to update it.") + elif provider_errors: + print() + print(f"{provider_errors} provider error(s) — hash not recorded; fix the setup and rerun.") print() print(f"Results report: {RESULTS_FILE.relative_to(REPO_ROOT)}") print( f"View results: PROMPTFOO_CONFIG_DIR={PROMPTFOO_STATE_DIR.relative_to(REPO_ROOT)}" - f" {' '.join(promptfoo_cmd)} view" + f" npx promptfoo@{PROMPTFOO_VERSION} view" ) return result.returncode diff --git a/scripts/ci/prek/check_eval_hash.py b/scripts/ci/prek/check_eval_hash.py index 74d2b3c40c9e2..540338807e994 100755 --- a/scripts/ci/prek/check_eval_hash.py +++ b/scripts/ci/prek/check_eval_hash.py @@ -89,7 +89,6 @@ def main() -> int: " skill-eval\n(dev/skill-evals/last-eval-hash.txt does not match).\n" "\n" " Run: prek run run-skill-eval --hook-stage manual --all-files\n" - " (or: uv run dev/skill-evals/eval.py)\n" " then commit the updated dev/skill-evals/last-eval-hash.txt\n" "\n" " WIP commit: SKIP=check-eval-hash git commit ...\n" diff --git a/scripts/tests/ci/prek/test_check_eval_hash.py b/scripts/tests/ci/prek/test_check_eval_hash.py index a811101752175..d3aeec4129eaa 100644 --- a/scripts/tests/ci/prek/test_check_eval_hash.py +++ b/scripts/tests/ci/prek/test_check_eval_hash.py @@ -125,5 +125,5 @@ def test_main_fails_on_mismatch_or_missing_record(guidance, tmp_path, capsys, re assert main() == 1 err = capsys.readouterr().err - assert "uv run dev/skill-evals/eval.py" in err + assert "prek run run-skill-eval" in err assert "SKIP=check-eval-hash" in err From 880474b1ee6ad4ff00715667ffc8679e8fed070f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=96=86=E5=AE=B8?= Date: Sun, 5 Jul 2026 20:23:06 +0900 Subject: [PATCH 16/16] Add positive newsfragment cases and view-skill-eval hook - 62343: merged with 62343.feature.rst, ground truth should_create=true - 65958: proposed as a should-create example by its author in review - view-skill-eval manual hook reuses run-skill-eval's node env --- .pre-commit-config.yaml | 13 +++++++++++++ dev/skill-evals/README.md | 4 ++-- dev/skill-evals/cases/newsfragment.yaml | 25 +++++++++++++++++++++++++ dev/skill-evals/last-eval-hash.txt | 2 +- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2386a95042019..c5f8e587ddbdc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1022,6 +1022,19 @@ repos: pass_filenames: false require_serial: true verbose: true + - id: view-skill-eval + name: View skill-eval results in browser (manual) + entry: env PROMPTFOO_CONFIG_DIR=.build/promptfoo promptfoo view + language: node + language_version: '22.22.0' + additional_dependencies: ['promptfoo@0.121.17', '@anthropic-ai/claude-agent-sdk@0.3.185'] + stages: ['manual'] + files: > + (?x) + ^AGENTS\.md$| + ^dev/skill-evals/ + pass_filenames: false + require_serial: true - id: check-eval-hash name: Verify skill-eval ran on current AGENTS.md and cases entry: ./scripts/ci/prek/check_eval_hash.py diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 18843322675dd..57748b96c2e5c 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -75,8 +75,8 @@ MODEL=claude-haiku-4-5-20251001 prek run run-skill-eval --hook-stage manual --al # baseline arm has no skill, so the skill-used assertion would always fail): SKILL_NAME=airflow-contribution prek run run-skill-eval --hook-stage manual --all-files -# View results in browser (state lives under .build/promptfoo): -PROMPTFOO_CONFIG_DIR=.build/promptfoo npx promptfoo@0.121.17 view +# View results in browser (Ctrl-C to stop the server): +prek run view-skill-eval --hook-stage manual --all-files ``` Other promptfoo flags (`--filter*`, `--no-cache`) are argv-only — diff --git a/dev/skill-evals/cases/newsfragment.yaml b/dev/skill-evals/cases/newsfragment.yaml index 7b0d4e630fa26..b769afa54864a 100644 --- a/dev/skill-evals/cases/newsfragment.yaml +++ b/dev/skill-evals/cases/newsfragment.yaml @@ -62,3 +62,28 @@ assert: - type: javascript value: 'output.should_create === false' + +- description: "Worker connection testing (#62343): feature newsfragment" + vars: + request: | + I added asynchronous connection testing: connection tests can now + run on a worker instead of in-process on the API server, submitted + via a new POST /connections/enqueue-test endpoint and configured + under a new [connection_test] config section. The changes span + airflow-core and task-sdk. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === true' + +- description: "Coordinator layer (#65958): feature newsfragment" + vars: + request: | + I added a coordinator layer to the task execution supervisor so + coordinators written in other languages can drive task execution, + along with a first Java coordinator and a new config option. Most + of the changes are in task-sdk, with some in airflow-core. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === true' diff --git a/dev/skill-evals/last-eval-hash.txt b/dev/skill-evals/last-eval-hash.txt index 6d6a5961c51c7..d0863d6365794 100644 --- a/dev/skill-evals/last-eval-hash.txt +++ b/dev/skill-evals/last-eval-hash.txt @@ -1,3 +1,3 @@ # Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. # Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate. -f959b6005d0efc11a2de0030b25b1f804788fd50118ba652ac2f7c71dfaf1d38 +f4a71898d35b99e6c42ce756a71fa24ba243ada0611ed55966fce25711ac8c02