From 988524bc61b92a1ca84e17fba420396b517a90a8 Mon Sep 17 00:00:00 2001 From: Justin McLean Date: Sat, 4 Jul 2026 18:39:45 +1000 Subject: [PATCH] test(spec-loop): add runner fixture coverage Generated-by: Claude (Opus 4.7) --- tools/spec-loop/lib.sh | 124 ++++++++++++++++ tools/spec-loop/loop.sh | 110 ++++---------- tools/spec-loop/specs/spec-loop-runner.md | 10 +- tools/spec-loop/tests/test_runner_fixtures.sh | 134 ++++++++++++++++++ .../src/spec_validator/__init__.py | 31 ++++ .../tests/test_spec_validator.py | 22 +++ 6 files changed, 346 insertions(+), 85 deletions(-) create mode 100644 tools/spec-loop/lib.sh create mode 100755 tools/spec-loop/tests/test_runner_fixtures.sh diff --git a/tools/spec-loop/lib.sh b/tools/spec-loop/lib.sh new file mode 100644 index 00000000..dfd2bec2 --- /dev/null +++ b/tools/spec-loop/lib.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Deterministic helpers for tools/spec-loop/loop.sh. Keep this file free of +# live agent execution so fixture tests can pin runner behaviour without +# launching a headless harness. + +spec_loop_append_tooling_source_context() { + local mode=$1 base=$2 tooling_ref=$3 + + cat < +\`\`\` + +EOF + if [ "$mode" = "update" ]; then + cat < "$dest" + cat "$compact_file" >> "$dest" + if [ "$mode" != "update" ]; then + cat "$pr_file" >> "$dest" + fi + cat "$branch_file" >> "$dest" + + if [ "$build_iteration" = "true" ] && [ "$tooling_ref" != "$base" ]; then + spec_loop_append_tooling_source_context "$mode" "$base" "$tooling_ref" >> "$dest" + fi + if [ "$mode" = "update" ] && [ -n "$update_scope_file" ]; then + cat "$update_scope_file" >> "$dest" + fi +} + +spec_loop_write_last_sync_marker() { + local marker=$1 base_head=$2 + mkdir -p "$(dirname "$marker")" + printf '%s\n' "$base_head" > "$marker" +} + +spec_loop_marker_branch_name() { + local base_head=$1 + printf 'advance-last-sync-%s\n' "${base_head:0:7}" +} + +spec_loop_launch_agent() { + local harness=$1 agent=$2 root=$3 prompt_file=$4 model=$5 output_format=$6 + local model_args=() + [ -n "$model" ] && model_args=(--model "$model") + + if [ "$harness" = "opencode" ]; then + local oc_format_args=() + [ "$output_format" = "stream-json" ] && oc_format_args=(--format json) + "$agent" run \ + --auto \ + ${model_args[@]+"${model_args[@]}"} \ + ${oc_format_args[@]+"${oc_format_args[@]}"} \ + "$(cat "$prompt_file")" & + elif [ "$harness" = "codex" ]; then + local codex_format_args=() + [ "$output_format" = "stream-json" ] && codex_format_args=(--json) + "$agent" exec \ + --dangerously-bypass-approvals-and-sandbox \ + --cd "$root" \ + ${model_args[@]+"${model_args[@]}"} \ + ${codex_format_args[@]+"${codex_format_args[@]}"} \ + - < "$prompt_file" & + elif [ "$harness" = "cursor" ]; then + local cursor_format_args=(--output-format "$output_format") + local cursor_subcommand=() + [ "$(basename "$agent")" = "cursor" ] && cursor_subcommand=(agent) + "$agent" \ + ${cursor_subcommand[@]+"${cursor_subcommand[@]}"} \ + --print \ + --force \ + --trust \ + --workspace "$root" \ + ${model_args[@]+"${model_args[@]}"} \ + ${cursor_format_args[@]+"${cursor_format_args[@]}"} \ + "$(cat "$prompt_file")" & + elif [ "$harness" = "gemini" ]; then + "$agent" \ + --yolo \ + ${model_args[@]+"${model_args[@]}"} \ + --prompt "$(cat "$prompt_file")" & + else + local verbose_args=() + [ "$output_format" = "stream-json" ] && verbose_args=(--verbose) + "$agent" -p \ + --dangerously-skip-permissions \ + --disallowedTools "Bash(git push:*)" "Bash(gh:*)" \ + --output-format="$output_format" \ + ${verbose_args[@]+"${verbose_args[@]}"} \ + ${model_args[@]+"${model_args[@]}"} < "$prompt_file" & + fi + SPEC_LOOP_AGENT_PID=$! +} diff --git a/tools/spec-loop/loop.sh b/tools/spec-loop/loop.sh index f2150fd8..8b763497 100755 --- a/tools/spec-loop/loop.sh +++ b/tools/spec-loop/loop.sh @@ -72,6 +72,8 @@ cd "$ROOT" || exit 1 LOOP_DIR="tools/spec-loop" PLAN="$LOOP_DIR/IMPLEMENTATION_PLAN.md" +# shellcheck source=tools/spec-loop/lib.sh +. "$LOOP_DIR/lib.sh" current_branch() { local branch @@ -400,19 +402,24 @@ while true; do # fall back to the control branch ($TOOLING_REF) if the tree is on a base # that lacks the tooling. Either way the read never breaks after checkout. PROMPT_WITH_CONTEXT="$(mktemp "${TMPDIR:-/tmp}/spec-loop-prompt.XXXXXX")" || exit 1 + PROMPT_SOURCE="$(mktemp "${TMPDIR:-/tmp}/spec-loop-source.XXXXXX")" || exit 1 if [ -f "$ACTIVE_PROMPT" ]; then - cat "$ACTIVE_PROMPT" > "$PROMPT_WITH_CONTEXT" - elif ! git show "$TOOLING_REF:$ACTIVE_PROMPT" > "$PROMPT_WITH_CONTEXT" 2>/dev/null; then + cat "$ACTIVE_PROMPT" > "$PROMPT_SOURCE" + elif ! git show "$TOOLING_REF:$ACTIVE_PROMPT" > "$PROMPT_SOURCE" 2>/dev/null; then echo "Error: could not read '$ACTIVE_PROMPT' from the working tree or control branch '$TOOLING_REF'." >&2 - rm -f "$PROMPT_WITH_CONTEXT"; break + rm -f "$PROMPT_WITH_CONTEXT" "$PROMPT_SOURCE"; break fi - compact_inventory_context >> "$PROMPT_WITH_CONTEXT" + COMPACT_CONTEXT="$(mktemp "${TMPDIR:-/tmp}/spec-loop-compact.XXXXXX")" || exit 1 + PR_CONTEXT="$(mktemp "${TMPDIR:-/tmp}/spec-loop-prs.XXXXXX")" || exit 1 + BRANCH_CONTEXT="$(mktemp "${TMPDIR:-/tmp}/spec-loop-branches.XXXXXX")" || exit 1 + UPDATE_SCOPE_CONTEXT="$(mktemp "${TMPDIR:-/tmp}/spec-loop-update-scope.XXXXXX")" || exit 1 + compact_inventory_context > "$COMPACT_CONTEXT" # Update mode just diffs code against specs; it doesn't pick a work item, so # the open-PR list (a network round-trip via gh) buys nothing. Skip it there. if [ "$MODE" != "update" ]; then - open_pr_context >> "$PROMPT_WITH_CONTEXT" + open_pr_context > "$PR_CONTEXT" fi - local_branch_context >> "$PROMPT_WITH_CONTEXT" + local_branch_context > "$BRANCH_CONTEXT" if [ "$BUILD_ITERATION" = true ]; then # The work-item branch forks off BASE (e.g. main), which need not @@ -454,7 +461,7 @@ while true; do echo "Error: could not check out base '$BASE'. git reported:" >&2 printf ' %s\n' "$checkout_out" >&2 echo "Resolve the working tree (commit or stash changes), then re-run." >&2 - rm -f "$PROMPT_WITH_CONTEXT"; break + rm -f "$PROMPT_WITH_CONTEXT" "$PROMPT_SOURCE" "$COMPACT_CONTEXT" "$PR_CONTEXT" "$BRANCH_CONTEXT" "$UPDATE_SCOPE_CONTEXT"; break fi fi BASE_HEAD="$(git rev-parse HEAD)" @@ -463,10 +470,14 @@ while true; do # and BASE_HEAD is known. The agent will diff $prev..$BASE_HEAD on the # listed paths and skip anything not touched in that range. if [ "$MODE" = "update" ]; then - update_scope_context >> "$PROMPT_WITH_CONTEXT" + update_scope_context > "$UPDATE_SCOPE_CONTEXT" fi fi + spec_loop_assemble_prompt_file \ + "$PROMPT_SOURCE" "$PROMPT_WITH_CONTEXT" "$MODE" "$BUILD_ITERATION" "$BASE" "$TOOLING_REF" \ + "$COMPACT_CONTEXT" "$PR_CONTEXT" "$BRANCH_CONTEXT" "$UPDATE_SCOPE_CONTEXT" + # Run one iteration with a fresh context. # -p headless / non-interactive # --dangerously-skip-permissions let the agent edit + validate @@ -478,81 +489,14 @@ while true; do # remote even with permissions skipped. # Claude CLI requires --verbose with -p when output-format is stream-json; # add it only in that case to keep the default `text` run quiet. - MODEL_ARGS=() - [ -n "$MODEL" ] && MODEL_ARGS=(--model "$MODEL") - if [ "$HARNESS" = "opencode" ]; then - # OpenCode headless: `opencode run ` takes the prompt as a - # positional argument (no stdin), `--model provider/model`, `--auto` - # auto-approves permissions not explicitly denied (the OpenCode - # equivalent of Claude's --dangerously-skip-permissions), and - # `--format json` matches the stream-json request. OpenCode has no - # per-invocation --disallowedTools; the push/gh denial that flag - # provides as defense-in-depth is instead enforced by the OS sandbox - # (the real guard — see the SECURITY header) plus, when wired, the - # agent-guard OpenCode plugin and the project's opencode.json - # `permission` config. - OC_FORMAT_ARGS=() - [ "$OUTPUT_FORMAT" = "stream-json" ] && OC_FORMAT_ARGS=(--format json) - "$AGENT" run \ - --auto \ - ${MODEL_ARGS[@]+"${MODEL_ARGS[@]}"} \ - ${OC_FORMAT_ARGS[@]+"${OC_FORMAT_ARGS[@]}"} \ - "$(cat "$PROMPT_WITH_CONTEXT")" & - elif [ "$HARNESS" = "codex" ]; then - # Codex CLI headless: `codex exec -` reads the prompt from stdin. - # `--dangerously-bypass-approvals-and-sandbox` is Codex's unattended - # automation switch; as with the other harnesses, only use this loop - # inside the external OS sandbox described in the SECURITY header. - # Codex has no per-invocation --disallowedTools equivalent here; the - # hard boundary is the OS sandbox plus repo hooks / exec policy. - CODEX_FORMAT_ARGS=() - [ "$OUTPUT_FORMAT" = "stream-json" ] && CODEX_FORMAT_ARGS=(--json) - "$AGENT" exec \ - --dangerously-bypass-approvals-and-sandbox \ - --cd "$ROOT" \ - ${MODEL_ARGS[@]+"${MODEL_ARGS[@]}"} \ - ${CODEX_FORMAT_ARGS[@]+"${CODEX_FORMAT_ARGS[@]}"} \ - - < "$PROMPT_WITH_CONTEXT" & - elif [ "$HARNESS" = "cursor" ]; then - # Cursor Agent headless: `cursor agent --print ` or the - # standalone `cursor-agent --print `. `--force` is Cursor's - # unattended "allow commands unless explicitly denied" mode, and - # `--trust` avoids a workspace-trust prompt in headless runs. - CURSOR_FORMAT_ARGS=(--output-format "$OUTPUT_FORMAT") - CURSOR_SUBCOMMAND=() - [ "$(basename "$AGENT")" = "cursor" ] && CURSOR_SUBCOMMAND=(agent) - "$AGENT" \ - ${CURSOR_SUBCOMMAND[@]+"${CURSOR_SUBCOMMAND[@]}"} \ - --print \ - --force \ - --trust \ - --workspace "$ROOT" \ - ${MODEL_ARGS[@]+"${MODEL_ARGS[@]}"} \ - ${CURSOR_FORMAT_ARGS[@]+"${CURSOR_FORMAT_ARGS[@]}"} \ - "$(cat "$PROMPT_WITH_CONTEXT")" & - elif [ "$HARNESS" = "gemini" ]; then - # Gemini CLI headless: `gemini --prompt ` runs - # non-interactively; `--yolo` automatically approves tool calls. - "$AGENT" \ - --yolo \ - ${MODEL_ARGS[@]+"${MODEL_ARGS[@]}"} \ - --prompt "$(cat "$PROMPT_WITH_CONTEXT")" & - else - # Claude Code headless (the default). - VERBOSE_ARGS=() - [ "$OUTPUT_FORMAT" = "stream-json" ] && VERBOSE_ARGS=(--verbose) - "$AGENT" -p \ - --dangerously-skip-permissions \ - --disallowedTools "Bash(git push:*)" "Bash(gh:*)" \ - --output-format="$OUTPUT_FORMAT" \ - ${VERBOSE_ARGS[@]+"${VERBOSE_ARGS[@]}"} \ - ${MODEL_ARGS[@]+"${MODEL_ARGS[@]}"} < "$PROMPT_WITH_CONTEXT" & - fi - AGENT_PID=$! + # Harness-specific launch details live in lib.sh so fixture tests can + # validate argv construction without starting an agent. + spec_loop_launch_agent "$HARNESS" "$AGENT" "$ROOT" "$PROMPT_WITH_CONTEXT" "$MODEL" "$OUTPUT_FORMAT" + AGENT_PID=$SPEC_LOOP_AGENT_PID spinner "$AGENT_PID" & SPINNER_PID=$! wait "$AGENT_PID" wait "$SPINNER_PID" 2>/dev/null - rm -f "$PROMPT_WITH_CONTEXT" + rm -f "$PROMPT_WITH_CONTEXT" "$PROMPT_SOURCE" "$COMPACT_CONTEXT" "$PR_CONTEXT" "$BRANCH_CONTEXT" "$UPDATE_SCOPE_CONTEXT" CUR_BRANCH="$(current_branch)" LAST_COMMIT="$(git log --oneline -1 2>/dev/null)" @@ -576,7 +520,7 @@ while true; do # on BASE — make a tiny marker-only branch off BASE. if [ "$MODE" = "update" ]; then if [ "$CUR_BRANCH" != "$BASE" ] && [ "$CUR_BRANCH" != "$TOOLING_REF" ]; then - printf '%s\n' "$BASE_HEAD" > tools/spec-loop/.last-sync + spec_loop_write_last_sync_marker tools/spec-loop/.last-sync "$BASE_HEAD" if ! git diff --quiet -- tools/spec-loop/.last-sync 2>/dev/null; then git add tools/spec-loop/.last-sync if git commit --amend --no-edit >/dev/null 2>&1; then @@ -589,9 +533,9 @@ while true; do cur_marker="" [ -f tools/spec-loop/.last-sync ] && cur_marker="$(tr -d '[:space:]' < tools/spec-loop/.last-sync)" if [ "$cur_marker" != "$BASE_HEAD" ]; then - marker_branch="advance-last-sync-${BASE_HEAD:0:7}" + marker_branch="$(spec_loop_marker_branch_name "$BASE_HEAD")" if git checkout -b "$marker_branch" >/dev/null 2>&1; then - printf '%s\n' "$BASE_HEAD" > tools/spec-loop/.last-sync + spec_loop_write_last_sync_marker tools/spec-loop/.last-sync "$BASE_HEAD" git add tools/spec-loop/.last-sync if git commit -m "chore(spec-loop): advance .last-sync to $BASE_HEAD" >/dev/null 2>&1; then echo "[ marker ] advanced .last-sync on $marker_branch" diff --git a/tools/spec-loop/specs/spec-loop-runner.md b/tools/spec-loop/specs/spec-loop-runner.md index 6662a6f3..1efdbd43 100644 --- a/tools/spec-loop/specs/spec-loop-runner.md +++ b/tools/spec-loop/specs/spec-loop-runner.md @@ -8,6 +8,7 @@ kind: feature mode: infra source: > tools/spec-loop/README.md, tools/spec-loop/loop.sh, + tools/spec-loop/lib.sh, tools/spec-loop/tests/test_runner_fixtures.sh, tools/spec-loop/PROMPT_plan.md, tools/spec-loop/PROMPT_build.md, tools/spec-loop/PROMPT_update.md, tools/spec-loop/PROMPT_consolidate.md, and docs/spec-driven-development.md. @@ -41,6 +42,12 @@ and never opens a pull request. ## Where it lives - `tools/spec-loop/loop.sh` — Bash runner for the four beats. +- `tools/spec-loop/lib.sh` — deterministic prompt assembly, harness + command rendering, agent launch, and `.last-sync` marker helpers used + by the runner and fixture tests. +- `tools/spec-loop/tests/test_runner_fixtures.sh` — deterministic + fixture tests for prompt assembly, harness command construction, and + `.last-sync` marker helpers; executed by `spec-validate`. - `tools/spec-loop/PROMPT_plan.md` — gap-analysis prompt that rewrites `IMPLEMENTATION_PLAN.md`. - `tools/spec-loop/PROMPT_build.md` — implementation prompt for exactly @@ -140,13 +147,12 @@ updating `loop.sh` in the same change. ```bash bash -n tools/spec-loop/loop.sh shellcheck tools/spec-loop/loop.sh +shellcheck tools/spec-loop/lib.sh tools/spec-loop/tests/test_runner_fixtures.sh uv run --project tools/spec-validator --group dev spec-validate ``` ## Known gaps -- There is no deterministic fixture test for prompt assembly, harness - command construction, or `.last-sync` marker amendment. - The non-Claude harnesses rely on external policy/config plus the OS sandbox for push/PR denial; only Claude has a per-invocation hard-deny flag in the current runner. diff --git a/tools/spec-loop/tests/test_runner_fixtures.sh b/tools/spec-loop/tests/test_runner_fixtures.sh new file mode 100755 index 00000000..20dda4e7 --- /dev/null +++ b/tools/spec-loop/tests/test_runner_fixtures.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +# shellcheck source=tools/spec-loop/lib.sh +. "$ROOT/tools/spec-loop/lib.sh" + +TMPDIR_TEST="$(mktemp -d "${TMPDIR:-/tmp}/spec-loop-fixtures.XXXXXX")" +trap 'rm -rf "$TMPDIR_TEST"' EXIT + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +assert_contains() { + local file=$1 expected=$2 + grep -Fq -- "$expected" "$file" || fail "expected '$file' to contain: $expected" +} + +assert_not_contains() { + local file=$1 unexpected=$2 + ! grep -Fq -- "$unexpected" "$file" || fail "expected '$file' not to contain: $unexpected" +} + +write_fixture() { + local file=$1 text=$2 + printf '%s\n' "$text" > "$file" +} + +test_prompt_assembly_build() { + write_fixture "$TMPDIR_TEST/prompt.md" "PROMPT BODY" + write_fixture "$TMPDIR_TEST/compact.md" "COMPACT INVENTORY" + write_fixture "$TMPDIR_TEST/prs.md" "OPEN PR CONTEXT" + write_fixture "$TMPDIR_TEST/branches.md" "LOCAL BRANCH CONTEXT" + write_fixture "$TMPDIR_TEST/update.md" "UPDATE SCOPE" + + spec_loop_assemble_prompt_file \ + "$TMPDIR_TEST/prompt.md" "$TMPDIR_TEST/build.out" build true main control-branch \ + "$TMPDIR_TEST/compact.md" "$TMPDIR_TEST/prs.md" "$TMPDIR_TEST/branches.md" "$TMPDIR_TEST/update.md" + + assert_contains "$TMPDIR_TEST/build.out" "PROMPT BODY" + assert_contains "$TMPDIR_TEST/build.out" "COMPACT INVENTORY" + assert_contains "$TMPDIR_TEST/build.out" "OPEN PR CONTEXT" + assert_contains "$TMPDIR_TEST/build.out" "LOCAL BRANCH CONTEXT" + assert_contains "$TMPDIR_TEST/build.out" "git show control-branch:tools/spec-loop/IMPLEMENTATION_PLAN.md" + assert_contains "$TMPDIR_TEST/build.out" "Implement the product change on the work branch" + assert_not_contains "$TMPDIR_TEST/build.out" "UPDATE SCOPE" +} + +test_prompt_assembly_update() { + write_fixture "$TMPDIR_TEST/prompt.md" "PROMPT BODY" + write_fixture "$TMPDIR_TEST/compact.md" "COMPACT INVENTORY" + write_fixture "$TMPDIR_TEST/prs.md" "OPEN PR CONTEXT" + write_fixture "$TMPDIR_TEST/branches.md" "LOCAL BRANCH CONTEXT" + write_fixture "$TMPDIR_TEST/update.md" "UPDATE SCOPE" + + spec_loop_assemble_prompt_file \ + "$TMPDIR_TEST/prompt.md" "$TMPDIR_TEST/update.out" update true main control-branch \ + "$TMPDIR_TEST/compact.md" "$TMPDIR_TEST/prs.md" "$TMPDIR_TEST/branches.md" "$TMPDIR_TEST/update.md" + + assert_contains "$TMPDIR_TEST/update.out" "PROMPT BODY" + assert_contains "$TMPDIR_TEST/update.out" "COMPACT INVENTORY" + assert_contains "$TMPDIR_TEST/update.out" "LOCAL BRANCH CONTEXT" + assert_contains "$TMPDIR_TEST/update.out" "Read the current specs from" + assert_contains "$TMPDIR_TEST/update.out" "UPDATE SCOPE" + assert_not_contains "$TMPDIR_TEST/update.out" "OPEN PR CONTEXT" +} + +make_fake_agent() { + local path=$1 log=$2 + cat > "$path" <' "\$@" + printf '\\nstdin:' + cat + printf '\\n' +} > "$log" +EOF + chmod +x "$path" +} + +test_harness_command_construction() { + write_fixture "$TMPDIR_TEST/prompt.md" "PROMPT BODY" + make_fake_agent "$TMPDIR_TEST/claude" "$TMPDIR_TEST/claude.log" + make_fake_agent "$TMPDIR_TEST/codex" "$TMPDIR_TEST/codex.log" + + spec_loop_launch_agent claude "$TMPDIR_TEST/claude" /repo "$TMPDIR_TEST/prompt.md" sonnet stream-json + wait "$SPEC_LOOP_AGENT_PID" + assert_contains "$TMPDIR_TEST/claude.log" "<-p>" + assert_contains "$TMPDIR_TEST/claude.log" "<--dangerously-skip-permissions>" + assert_contains "$TMPDIR_TEST/claude.log" "<--disallowedTools>" + assert_contains "$TMPDIR_TEST/claude.log" "" + assert_contains "$TMPDIR_TEST/claude.log" "" + assert_contains "$TMPDIR_TEST/claude.log" "<--output-format=stream-json>" + assert_contains "$TMPDIR_TEST/claude.log" "<--verbose>" + assert_contains "$TMPDIR_TEST/claude.log" "<--model>" + assert_contains "$TMPDIR_TEST/claude.log" "" + assert_contains "$TMPDIR_TEST/claude.log" "stdin:PROMPT BODY" + + spec_loop_launch_agent codex "$TMPDIR_TEST/codex" /repo "$TMPDIR_TEST/prompt.md" gpt-5 stream-json + wait "$SPEC_LOOP_AGENT_PID" + assert_contains "$TMPDIR_TEST/codex.log" "" + assert_contains "$TMPDIR_TEST/codex.log" "<--dangerously-bypass-approvals-and-sandbox>" + assert_contains "$TMPDIR_TEST/codex.log" "<--cd>" + assert_contains "$TMPDIR_TEST/codex.log" "" + assert_contains "$TMPDIR_TEST/codex.log" "<--model>" + assert_contains "$TMPDIR_TEST/codex.log" "" + assert_contains "$TMPDIR_TEST/codex.log" "<--json>" + assert_contains "$TMPDIR_TEST/codex.log" "<->" + assert_contains "$TMPDIR_TEST/codex.log" "stdin:PROMPT BODY" +} + +test_last_sync_marker_helpers() { + local marker="$TMPDIR_TEST/.last-sync" + spec_loop_write_last_sync_marker "$marker" "abcdef1234567890" + assert_contains "$marker" "abcdef1234567890" + + local branch + branch="$(spec_loop_marker_branch_name "abcdef1234567890")" + [ "$branch" = "advance-last-sync-abcdef1" ] || fail "unexpected marker branch: $branch" +} + +test_prompt_assembly_build +test_prompt_assembly_update +test_harness_command_construction +test_last_sync_marker_helpers + +printf 'spec-loop runner fixtures: OK\n' diff --git a/tools/spec-validator/src/spec_validator/__init__.py b/tools/spec-validator/src/spec_validator/__init__.py index ad37adcb..c409f181 100644 --- a/tools/spec-validator/src/spec_validator/__init__.py +++ b/tools/spec-validator/src/spec_validator/__init__.py @@ -35,6 +35,9 @@ shell pattern (``--project``, ``--directory``, ``bash -n``, ``shellcheck``, ``test -f``) in a Validation code block must exist under the repository root. Catches stale paths after renames. +10. Spec-loop runner fixtures — when present, execute the deterministic + Bash fixture suite for prompt assembly, harness argv construction, and + ``.last-sync`` marker helpers. Files without frontmatter (README.md, overview.md) are skipped silently. @@ -48,6 +51,7 @@ import argparse import re +import subprocess import sys from pathlib import Path @@ -447,9 +451,36 @@ def run_validation(target: Path, repo_root: Path | None = None) -> list[Violatio violations: list[Violation] = [] for path in collect_spec_files(target): violations.extend(validate_file(path, repo_root=root)) + violations.extend(validate_spec_loop_runner_fixtures(root)) return violations +def validate_spec_loop_runner_fixtures(repo_root: Path) -> list[Violation]: + """Run deterministic fixture tests for the spec-loop runner when present.""" + fixture = repo_root / "tools" / "spec-loop" / "tests" / "test_runner_fixtures.sh" + if not fixture.exists(): + return [] + + result = subprocess.run( + ["bash", str(fixture)], + cwd=repo_root, + text=True, + capture_output=True, + check=False, + ) + if result.returncode == 0: + return [] + + output = "\n".join(part for part in (result.stdout.strip(), result.stderr.strip()) if part) + return [ + Violation( + fixture.relative_to(repo_root), + None, + f"spec-loop runner fixture tests failed with exit code {result.returncode}: {output}", + ) + ] + + # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- diff --git a/tools/spec-validator/tests/test_spec_validator.py b/tools/spec-validator/tests/test_spec_validator.py index d33c6491..861039f5 100644 --- a/tools/spec-validator/tests/test_spec_validator.py +++ b/tools/spec-validator/tests/test_spec_validator.py @@ -40,6 +40,7 @@ validate_body, validate_frontmatter, validate_spdx_header, + validate_spec_loop_runner_fixtures, validate_validation_paths, validation_has_code_block, ) @@ -467,6 +468,27 @@ def test_main_violations(self, tmp_path: Path, capsys: pytest.CaptureFixture[str assert "violation" in captured.out +class TestSpecLoopRunnerFixtures: + def test_missing_fixture_is_skipped(self, tmp_path: Path) -> None: + (tmp_path / "tools").mkdir() + assert validate_spec_loop_runner_fixtures(tmp_path) == [] + + def test_passing_fixture_is_ok(self, tmp_path: Path) -> None: + fixture = tmp_path / "tools" / "spec-loop" / "tests" / "test_runner_fixtures.sh" + fixture.parent.mkdir(parents=True) + fixture.write_text("#!/usr/bin/env bash\nexit 0\n") + assert validate_spec_loop_runner_fixtures(tmp_path) == [] + + def test_failing_fixture_is_reported(self, tmp_path: Path) -> None: + fixture = tmp_path / "tools" / "spec-loop" / "tests" / "test_runner_fixtures.sh" + fixture.parent.mkdir(parents=True) + fixture.write_text("#!/usr/bin/env bash\necho broken >&2\nexit 7\n") + violations = validate_spec_loop_runner_fixtures(tmp_path) + assert len(violations) == 1 + assert "exit code 7" in violations[0].message + assert "broken" in violations[0].message + + # --------------------------------------------------------------------------- # Validation-path check (check #8) # ---------------------------------------------------------------------------