From 067cc2673e6a7c650e59541c1eecd16ea7f915cd Mon Sep 17 00:00:00 2001 From: Justin McLean Date: Wed, 27 May 2026 01:52:30 +1000 Subject: [PATCH 1/3] =?UTF-8?q?feat(meta):=20add=20spec-validator=20tool?= =?UTF-8?q?=20=E2=80=94=20validate=20spec=20frontmatter=20and=20body=20sec?= =?UTF-8?q?tions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tools/spec-validator/, a stdlib-only uv tool analogous to tools/skill-validator/ that validates every .md file carrying a YAML frontmatter block in tools/spec-loop/specs/: 1. Required frontmatter keys (title, status, kind, mode, source, acceptance) 2. Valid status / kind / mode values 3. Non-empty acceptance list 4. Required body sections (What it does, Where it lives, Behaviour & contract, Out of scope, Acceptance criteria, Validation) 5. Validation section contains at least one fenced code block Files without frontmatter (README.md, overview.md) are silently skipped. 56 tests pass; 11 live specs from the control branch validate clean. Note: all 7 IMPLEMENTATION_PLAN.md items were found to be merged or in-flight before this iteration; this item was derived from the Known gap in specs/meta-and-quality-tooling.md ("a spec validator analogous to the skill validator"). A plan/update beat should reconcile. Generated-by: Claude (Opus 4.7) --- tools/spec-validator/README.md | 49 ++ tools/spec-validator/pyproject.toml | 58 +++ .../src/spec_validator/__init__.py | 346 +++++++++++++++ .../tests/test_spec_validator.py | 417 ++++++++++++++++++ tools/spec-validator/uv.lock | 112 +++++ 5 files changed, 982 insertions(+) create mode 100644 tools/spec-validator/README.md create mode 100644 tools/spec-validator/pyproject.toml create mode 100644 tools/spec-validator/src/spec_validator/__init__.py create mode 100644 tools/spec-validator/tests/test_spec_validator.py create mode 100644 tools/spec-validator/uv.lock diff --git a/tools/spec-validator/README.md b/tools/spec-validator/README.md new file mode 100644 index 00000000..c94e2915 --- /dev/null +++ b/tools/spec-validator/README.md @@ -0,0 +1,49 @@ + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [spec-validator](#spec-validator) + - [What it checks](#what-it-checks) + - [Usage](#usage) + + + + + +# spec-validator + +Validates spec files in `tools/spec-loop/specs/` — the counterpart to +`tools/skill-validator/` for the spec side of the framework. + +## What it checks + +For every `.md` file that carries a YAML frontmatter block: + +1. **Required frontmatter keys** — `title`, `status`, `kind`, `mode`, + `source`, `acceptance`. +2. **Valid `status`** — `stable` | `experimental` | `proposed` | `off`. +3. **Valid `kind`** — `feature` | `fix` | `docs` | `chore`. +4. **Valid `mode`** — `Triage` | `Mentoring` | `Drafting` | `Pairing` | `infra`. +5. **Non-empty `acceptance` list** — at least one `- item` entry. +6. **Required body sections** — `## What it does`, `## Where it lives`, + `## Behaviour & contract`, `## Out of scope`, `## Acceptance criteria`, + `## Validation`. +7. **Validation section has a fenced code block** — at least one `` ```…``` `` + block so build-loop backpressure commands are always explicit. + +Files without frontmatter (e.g. `README.md`, `overview.md`) are silently +skipped — they are index/overview docs, not functional specs. + +## Usage + +```bash +# Run against the default spec directory +uv run --project tools/spec-validator spec-validate + +# Run against a specific directory or file +uv run --project tools/spec-validator spec-validate tools/spec-loop/specs/ + +# Run the test suite +uv run --project tools/spec-validator --group dev pytest +``` diff --git a/tools/spec-validator/pyproject.toml b/tools/spec-validator/pyproject.toml new file mode 100644 index 00000000..0fd95bcd --- /dev/null +++ b/tools/spec-validator/pyproject.toml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "spec-validator" +version = "0.1.0" +description = "Validate spec files — YAML frontmatter and required body sections." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Apache-2.0" } +dependencies = [] + +[project.scripts] +spec-validate = "spec_validator:main" + +[dependency-groups] +dev = [ + "pytest>=8.0", + "ruff>=0.6", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/spec_validator"] + +[tool.ruff] +line-length = 110 +target-version = "py311" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "B", "UP", "SIM", "C4", "RUF"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["B", "SIM"] + +[tool.pytest.ini_options] +minversion = "8.0" +addopts = "-ra -q" +testpaths = ["tests"] diff --git a/tools/spec-validator/src/spec_validator/__init__.py b/tools/spec-validator/src/spec_validator/__init__.py new file mode 100644 index 00000000..7b5166e1 --- /dev/null +++ b/tools/spec-validator/src/spec_validator/__init__.py @@ -0,0 +1,346 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate spec files in tools/spec-loop/specs/. + +Checks every .md file that carries a YAML frontmatter block: + +1. Required frontmatter keys — title, status, kind, mode, source, acceptance. +2. Valid ``status`` value — stable | experimental | proposed | off. +3. Valid ``kind`` value — feature | fix | docs | chore. +4. Valid ``mode`` value — Triage | Mentoring | Drafting | Pairing | infra. +5. Non-empty ``acceptance`` list — at least one ``- item`` entry. +6. Required body sections — What it does, Where it lives, + Behaviour & contract, Out of scope, Acceptance criteria, Validation. +7. Validation section contains at least one fenced code block. + +Files without frontmatter (README.md, overview.md) are skipped silently. + +Run from repo root:: + + uv run --project tools/spec-validator --group dev pytest + uv run --project tools/spec-validator spec-validate tools/spec-loop/specs/ +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +REQUIRED_FRONTMATTER_KEYS: frozenset[str] = frozenset( + {"title", "status", "kind", "mode", "source", "acceptance"} +) +ALLOWED_STATUS: frozenset[str] = frozenset({"stable", "experimental", "proposed", "off"}) +ALLOWED_KIND: frozenset[str] = frozenset({"feature", "fix", "docs", "chore"}) +ALLOWED_MODE: frozenset[str] = frozenset({"Triage", "Mentoring", "Drafting", "Pairing", "infra"}) + +REQUIRED_SECTIONS: tuple[str, ...] = ( + "What it does", + "Where it lives", + "Behaviour & contract", + "Out of scope", + "Acceptance criteria", + "Validation", +) + +DEFAULT_SPEC_DIR = Path("tools/spec-loop/specs") + +_HTML_COMMENT_RE = re.compile(r"") +_FENCED_CODE_RE = re.compile(r"^ {0,3}```[\s\S]*?^ {0,3}```", re.MULTILINE) +_YAML_BLOCK_SCALAR_HEADERS: frozenset[str] = frozenset({"|", ">", "|-", "|+", ">-", ">+"}) + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +class Violation: + def __init__(self, path: Path, line: int | None, message: str) -> None: + self.path = path + self.line = line + self.message = message + + def __str__(self) -> str: + if self.line is not None: + return f"{self.path}:{self.line}: {self.message}" + return f"{self.path}: {self.message}" + + +# --------------------------------------------------------------------------- +# Frontmatter parsing +# --------------------------------------------------------------------------- + + +def _frontmatter_bounds(text: str) -> tuple[int, int] | None: + """Return (block_start, block_end) for the frontmatter content, or None. + + Handles files whose first non-whitespace content is an HTML comment + (e.g. the SPDX license header) before the ``---`` delimiter. + """ + idx = text.find("---\n") + if idx == -1: + return None + # Verify only HTML comments / whitespace precede the opening --- + prefix = text[:idx] + clean = _HTML_COMMENT_RE.sub("", prefix).strip() + if clean: + return None + try: + end = text.index("\n---\n", idx + 4) + except ValueError: + return None + return (idx + 4, end) + + +def parse_frontmatter(text: str) -> dict[str, str] | None: + """Return a dict of top-level frontmatter key→value, or None if absent.""" + bounds = _frontmatter_bounds(text) + if bounds is None: + return None + block = text[bounds[0] : bounds[1]] + + result: dict[str, str] = {} + current_key: str | None = None + current_value_lines: list[str] = [] + + for raw_line in block.splitlines(): + line = raw_line.rstrip() + if line == "": + if current_key is not None: + current_value_lines.append("") + continue + if not line.startswith((" ", "\t")) and ":" in line: + if current_key is not None: + result[current_key] = "\n".join(current_value_lines).strip() + key, _, value = line.partition(":") + current_key = key.strip() + inline = value.strip() + current_value_lines = [inline] if inline and inline not in _YAML_BLOCK_SCALAR_HEADERS else [] + continue + if current_key is not None: + stripped = line[2:] if line.startswith(" ") else line + current_value_lines.append(stripped) + + if current_key is not None: + result[current_key] = "\n".join(current_value_lines).strip() + return result + + +def has_acceptance_items(text: str) -> bool: + """Return True if the ``acceptance`` frontmatter key has at least one list item.""" + bounds = _frontmatter_bounds(text) + if bounds is None: + return False + block = text[bounds[0] : bounds[1]] + in_acceptance = False + for line in block.splitlines(): + if not line.startswith((" ", "\t")) and ":" in line: + in_acceptance = line.split(":", 1)[0].strip() == "acceptance" + continue + if in_acceptance and re.match(r"\s+-\s", line): + return True + return False + + +# --------------------------------------------------------------------------- +# Body section validation +# --------------------------------------------------------------------------- + + +def _spec_body(text: str) -> str: + """Return the doc body — everything after the closing ``---`` frontmatter delimiter.""" + bounds = _frontmatter_bounds(text) + if bounds is None: + return text + # body starts after "\n---\n" + return text[bounds[1] + 5 :] + + +def extract_section_headings(text: str) -> set[str]: + """Return the text of every ## heading in the spec body.""" + body = _spec_body(text) + headings: set[str] = set() + for line in body.splitlines(): + if line.startswith("## "): + headings.add(line[3:].strip()) + return headings + + +def get_section_body(text: str, section: str) -> str | None: + """Return the content of a named ## section, or None.""" + body = _spec_body(text) + lines = body.splitlines() + collecting = False + collected: list[str] = [] + for line in lines: + if line.startswith("## "): + heading = line[3:].strip() + if heading == section: + collecting = True + continue + if collecting: + break + if collecting: + collected.append(line) + return "\n".join(collected) if collected else None + + +def validation_has_code_block(text: str) -> bool: + """Return True if the Validation section contains at least one fenced code block.""" + section = get_section_body(text, "Validation") + if not section: + return False + return bool(_FENCED_CODE_RE.search(section)) + + +# --------------------------------------------------------------------------- +# Validators +# --------------------------------------------------------------------------- + + +def validate_frontmatter(path: Path, text: str) -> list[Violation]: + fm = parse_frontmatter(text) + if fm is None: + return [] # No frontmatter — not a spec file; skip silently + + violations: list[Violation] = [] + + missing = REQUIRED_FRONTMATTER_KEYS - set(fm.keys()) + for key in sorted(missing): + violations.append(Violation(path, 1, f"missing required frontmatter key: '{key}'")) + + if "status" in fm and fm["status"] not in ALLOWED_STATUS: + violations.append( + Violation( + path, + 1, + f"invalid status '{fm['status']}' — must be one of {sorted(ALLOWED_STATUS)}", + ) + ) + + if "kind" in fm and fm["kind"] not in ALLOWED_KIND: + violations.append( + Violation( + path, + 1, + f"invalid kind '{fm['kind']}' — must be one of {sorted(ALLOWED_KIND)}", + ) + ) + + if "mode" in fm and fm["mode"] not in ALLOWED_MODE: + violations.append( + Violation( + path, + 1, + f"invalid mode '{fm['mode']}' — must be one of {sorted(ALLOWED_MODE)}", + ) + ) + + if "acceptance" in fm and not has_acceptance_items(text): + violations.append( + Violation(path, 1, "acceptance key is present but has no list items (expected ' - ...')") + ) + + return violations + + +def validate_body(path: Path, text: str) -> list[Violation]: + if parse_frontmatter(text) is None: + return [] # Not a spec file + + violations: list[Violation] = [] + headings = extract_section_headings(text) + + for section in REQUIRED_SECTIONS: + if section not in headings: + violations.append(Violation(path, None, f"missing required section: '## {section}'")) + + if "Validation" in headings and not validation_has_code_block(text): + violations.append( + Violation(path, None, "Validation section has no fenced code block (expected ```...```)") + ) + + return violations + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def validate_file(path: Path) -> list[Violation]: + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + return [Violation(path, None, f"cannot read file: {exc}")] + return validate_frontmatter(path, text) + validate_body(path, text) + + +def collect_spec_files(target: Path) -> list[Path]: + """Return all .md files under *target* (or *target* itself if a file).""" + if target.is_file(): + return [target] + return sorted(target.rglob("*.md")) + + +def run_validation(target: Path) -> list[Violation]: + violations: list[Violation] = [] + for path in collect_spec_files(target): + violations.extend(validate_file(path)) + return violations + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Validate spec files.") + parser.add_argument( + "path", + nargs="?", + default=str(DEFAULT_SPEC_DIR), + help="Spec file or directory to validate (default: tools/spec-loop/specs/)", + ) + args = parser.parse_args(argv) + + target = Path(args.path) + if not target.exists(): + print(f"spec-validator: path not found: {target}", file=sys.stderr) + return 1 + + violations = run_validation(target) + if not violations: + print("spec-validator: OK (no violations)") + return 0 + + print(f"spec-validator: {len(violations)} violation(s) found\n") + for v in violations: + print(v) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/spec-validator/tests/test_spec_validator.py b/tools/spec-validator/tests/test_spec_validator.py new file mode 100644 index 00000000..d42caf9d --- /dev/null +++ b/tools/spec-validator/tests/test_spec_validator.py @@ -0,0 +1,417 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the spec validator.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from spec_validator import ( + ALLOWED_KIND, + ALLOWED_MODE, + ALLOWED_STATUS, + REQUIRED_FRONTMATTER_KEYS, + REQUIRED_SECTIONS, + extract_section_headings, + get_section_body, + has_acceptance_items, + main, + parse_frontmatter, + run_validation, + validate_body, + validate_file, + validate_frontmatter, + validation_has_code_block, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_VALID_SPEC = textwrap.dedent("""\ + + + --- + title: Example spec + status: stable + kind: feature + mode: Triage + source: MISSION.md § some section + acceptance: + - At least one criterion is met. + --- + + # Example spec + + ## What it does + + A brief description. + + ## Where it lives + + - `tools/example/` + + ## Behaviour & contract + + The contract. + + ## Out of scope + + Nothing. + + ## Acceptance criteria + + 1. Criterion one. + + ## Validation + + ```bash + uv run --project tools/example --group dev pytest + ``` + """) + + +def _make_spec(*, status: str = "stable", **overrides: str) -> str: + """Build a minimal valid spec, replacing frontmatter values as needed.""" + defaults = { + "title": "Test spec", + "kind": "feature", + "mode": "Triage", + "source": "MISSION.md", + "acceptance_items": " - One criterion.", + } + defaults.update(overrides) + acceptance_items = defaults.pop("acceptance_items") + fm_lines = [ + f"title: {defaults['title']}", + f"status: {status}", + f"kind: {defaults['kind']}", + f"mode: {defaults['mode']}", + f"source: {defaults['source']}", + "acceptance:", + acceptance_items, + ] + body_sections = "\n\n".join( + f"## {s}\n\nContent." for s in REQUIRED_SECTIONS + ) + # Replace Validation section with one that has a code block + body_sections = body_sections.replace( + "## Validation\n\nContent.", + "## Validation\n\n```bash\npytest\n```", + ) + fm = "\n".join(fm_lines) + return f"---\n{fm}\n---\n\n# Test spec\n\n{body_sections}\n" + + +# --------------------------------------------------------------------------- +# Frontmatter parsing +# --------------------------------------------------------------------------- + + +class TestParseFrontmatter: + def test_valid_frontmatter(self) -> None: + fm = parse_frontmatter(_VALID_SPEC) + assert fm is not None + assert fm["title"] == "Example spec" + assert fm["status"] == "stable" + + def test_no_frontmatter_returns_none(self) -> None: + assert parse_frontmatter("# Just a heading\n\nNo frontmatter.") is None + + def test_html_comment_prefix_allowed(self) -> None: + text = "\n---\ntitle: foo\n---\n" + fm = parse_frontmatter(text) + assert fm is not None + assert fm["title"] == "foo" + + def test_non_comment_prefix_returns_none(self) -> None: + text = "Some prose\n---\ntitle: foo\n---\n" + assert parse_frontmatter(text) is None + + def test_folded_block_scalar(self) -> None: + text = "---\nsource: >\n line one\n line two\n---\n" + fm = parse_frontmatter(text) + assert fm is not None + assert "line one" in fm["source"] + + def test_multiline_value(self) -> None: + text = "---\ntitle: My\n continuation\n---\n" + fm = parse_frontmatter(text) + assert fm is not None + assert "My" in fm["title"] + + +class TestHasAcceptanceItems: + def test_has_items(self) -> None: + text = "---\nacceptance:\n - item one\n - item two\n---\n" + assert has_acceptance_items(text) is True + + def test_no_items(self) -> None: + text = "---\nacceptance:\n---\n" + assert has_acceptance_items(text) is False + + def test_no_frontmatter(self) -> None: + assert has_acceptance_items("# no frontmatter") is False + + +# --------------------------------------------------------------------------- +# Body section extraction +# --------------------------------------------------------------------------- + + +class TestExtractSectionHeadings: + def test_extracts_h2_headings(self) -> None: + headings = extract_section_headings(_VALID_SPEC) + assert "What it does" in headings + assert "Validation" in headings + + def test_ignores_h1(self) -> None: + headings = extract_section_headings(_VALID_SPEC) + assert "Example spec" not in headings + + def test_no_frontmatter_still_works(self) -> None: + text = "# Title\n\n## Section A\n\ncontent\n" + headings = extract_section_headings(text) + assert "Section A" in headings + + +class TestGetSectionBody: + def test_returns_section_content(self) -> None: + body = get_section_body(_VALID_SPEC, "What it does") + assert body is not None + assert "A brief description" in body + + def test_returns_none_for_missing_section(self) -> None: + assert get_section_body(_VALID_SPEC, "Nonexistent") is None + + def test_stops_at_next_section(self) -> None: + body = get_section_body(_VALID_SPEC, "What it does") + assert body is not None + assert "Where it lives" not in body + + +class TestValidationHasCodeBlock: + def test_valid_spec_has_code_block(self) -> None: + assert validation_has_code_block(_VALID_SPEC) is True + + def test_missing_code_block(self) -> None: + text = _VALID_SPEC.replace("```bash\n uv run --project tools/example --group dev pytest\n ```", "Run pytest.") + # simpler replacement + spec = _make_spec() + spec_no_code = spec.replace("```bash\npytest\n```", "Run pytest manually.") + assert validation_has_code_block(spec_no_code) is False + + def test_no_validation_section(self) -> None: + text = "---\ntitle: t\n---\n## Other\n\ncontent\n" + assert validation_has_code_block(text) is False + + +# --------------------------------------------------------------------------- +# validate_frontmatter +# --------------------------------------------------------------------------- + + +class TestValidateFrontmatter: + def test_valid_spec_no_violations(self, tmp_path: Path) -> None: + p = tmp_path / "spec.md" + p.write_text(_VALID_SPEC) + assert validate_frontmatter(p, _VALID_SPEC) == [] + + def test_no_frontmatter_skipped(self, tmp_path: Path) -> None: + text = "# No frontmatter\n\ncontent\n" + p = tmp_path / "readme.md" + p.write_text(text) + assert validate_frontmatter(p, text) == [] + + def test_missing_required_key(self, tmp_path: Path) -> None: + text = "---\ntitle: foo\nstatus: stable\n---\n# foo\n" + p = tmp_path / "spec.md" + p.write_text(text) + violations = validate_frontmatter(p, text) + messages = [v.message for v in violations] + assert any("kind" in m for m in messages) + assert any("mode" in m for m in messages) + + @pytest.mark.parametrize("status", sorted(ALLOWED_STATUS)) + def test_all_valid_statuses_pass(self, tmp_path: Path, status: str) -> None: + text = _make_spec(status=status) + p = tmp_path / "spec.md" + p.write_text(text) + violations = [v for v in validate_frontmatter(p, text) if "status" in v.message] + assert violations == [] + + def test_invalid_status(self, tmp_path: Path) -> None: + text = _make_spec(status="unknown") + p = tmp_path / "spec.md" + p.write_text(text) + violations = validate_frontmatter(p, text) + assert any("invalid status" in v.message for v in violations) + + @pytest.mark.parametrize("kind", sorted(ALLOWED_KIND)) + def test_all_valid_kinds_pass(self, tmp_path: Path, kind: str) -> None: + text = _make_spec(kind=kind) + p = tmp_path / "spec.md" + violations = [v for v in validate_frontmatter(p, text) if "kind" in v.message] + assert violations == [] + + def test_invalid_kind(self, tmp_path: Path) -> None: + text = _make_spec(kind="unknown") + p = tmp_path / "spec.md" + violations = validate_frontmatter(p, text) + assert any("invalid kind" in v.message for v in violations) + + @pytest.mark.parametrize("mode", sorted(ALLOWED_MODE)) + def test_all_valid_modes_pass(self, tmp_path: Path, mode: str) -> None: + text = _make_spec(mode=mode) + p = tmp_path / "spec.md" + violations = [v for v in validate_frontmatter(p, text) if "mode" in v.message] + assert violations == [] + + def test_invalid_mode(self, tmp_path: Path) -> None: + text = _make_spec(mode="UnknownMode") + p = tmp_path / "spec.md" + violations = validate_frontmatter(p, text) + assert any("invalid mode" in v.message for v in violations) + + def test_empty_acceptance_list(self, tmp_path: Path) -> None: + text = "---\ntitle: t\nstatus: stable\nkind: feature\nmode: Triage\nsource: x\nacceptance:\n---\n# t\n" + p = tmp_path / "spec.md" + violations = validate_frontmatter(p, text) + assert any("acceptance" in v.message for v in violations) + + def test_acceptance_with_items_passes(self, tmp_path: Path) -> None: + text = _make_spec() + p = tmp_path / "spec.md" + violations = [v for v in validate_frontmatter(p, text) if "acceptance" in v.message] + assert violations == [] + + +# --------------------------------------------------------------------------- +# validate_body +# --------------------------------------------------------------------------- + + +class TestValidateBody: + def test_valid_spec_no_violations(self, tmp_path: Path) -> None: + p = tmp_path / "spec.md" + p.write_text(_VALID_SPEC) + assert validate_body(p, _VALID_SPEC) == [] + + def test_no_frontmatter_skipped(self, tmp_path: Path) -> None: + text = "# No frontmatter\n\n## What it does\n\ncontent\n" + p = tmp_path / "readme.md" + assert validate_body(p, text) == [] + + @pytest.mark.parametrize("section", REQUIRED_SECTIONS) + def test_missing_section_flagged(self, tmp_path: Path, section: str) -> None: + text = _make_spec() + # Remove the section heading + text_no_section = text.replace(f"## {section}\n", "## REPLACED_SECTION\n") + p = tmp_path / "spec.md" + violations = validate_body(p, text_no_section) + assert any(section in v.message for v in violations) + + def test_validation_without_code_block(self, tmp_path: Path) -> None: + text = _make_spec() + text_no_code = text.replace("```bash\npytest\n```", "Run manually.") + p = tmp_path / "spec.md" + violations = validate_body(p, text_no_code) + assert any("fenced code block" in v.message for v in violations) + + def test_all_sections_present_no_violations(self, tmp_path: Path) -> None: + text = _make_spec() + p = tmp_path / "spec.md" + assert validate_body(p, text) == [] + + +# --------------------------------------------------------------------------- +# run_validation (integration) +# --------------------------------------------------------------------------- + + +class TestRunValidation: + def test_valid_directory_no_violations(self, tmp_path: Path) -> None: + (tmp_path / "spec_a.md").write_text(_VALID_SPEC) + (tmp_path / "spec_b.md").write_text(_make_spec(status="experimental")) + assert run_validation(tmp_path) == [] + + def test_readme_skipped(self, tmp_path: Path) -> None: + (tmp_path / "README.md").write_text("# README\n\nNo frontmatter.\n") + assert run_validation(tmp_path) == [] + + def test_invalid_spec_produces_violations(self, tmp_path: Path) -> None: + text = "---\ntitle: broken\n---\n# broken\n" + (tmp_path / "broken.md").write_text(text) + violations = run_validation(tmp_path) + assert len(violations) > 0 + + def test_single_file_target(self, tmp_path: Path) -> None: + p = tmp_path / "spec.md" + p.write_text(_VALID_SPEC) + assert run_validation(p) == [] + + def test_nonexistent_path_via_main(self, capsys: pytest.CaptureFixture[str]) -> None: + rc = main(["/nonexistent/path"]) + assert rc == 1 + captured = capsys.readouterr() + assert "not found" in captured.err + + def test_main_ok(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "spec.md").write_text(_VALID_SPEC) + rc = main([str(tmp_path)]) + assert rc == 0 + captured = capsys.readouterr() + assert "OK" in captured.out + + def test_main_violations(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "bad.md").write_text("---\ntitle: bad\n---\n# bad\n") + rc = main([str(tmp_path)]) + assert rc == 1 + captured = capsys.readouterr() + assert "violation" in captured.out + + +# --------------------------------------------------------------------------- +# Live specs (smoke test) +# --------------------------------------------------------------------------- + + +class TestLiveSpecs: + """Run the validator against the actual specs on disk.""" + + @pytest.fixture + def specs_dir(self) -> Path | None: + """Locate tools/spec-loop/specs/ relative to the repo root.""" + start = Path(__file__).resolve() + for candidate in (start, *start.parents): + p = candidate / "tools" / "spec-loop" / "specs" + if p.is_dir(): + return p + return None + + def test_live_specs_pass(self, specs_dir: Path | None) -> None: + if specs_dir is None: + pytest.skip("tools/spec-loop/specs/ not found — skipping live test") + violations = run_validation(specs_dir) + if violations: + messages = "\n".join(str(v) for v in violations) + pytest.fail(f"Live spec violations found:\n{messages}") diff --git a/tools/spec-validator/uv.lock b/tools/spec-validator/uv.lock new file mode 100644 index 00000000..4abefef8 --- /dev/null +++ b/tools/spec-validator/uv.lock @@ -0,0 +1,112 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P7D" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/21/a7d5c126d5b557715ef81098f3db2fe20f622a039ff2e626af28d674ab80/ruff-0.15.13.tar.gz", hash = "sha256:f9d89f17f7ba7fb2ed42921f0df75da797a9a5d71bc39049e2c687cf2baf44b7", size = 4678180, upload-time = "2026-05-14T13:44:37.869Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/61/11d458dc6ac22504fd8e237b29dfd40504c7fbbcc8930402cfe51a8e63ed/ruff-0.15.13-py3-none-linux_armv6l.whl", hash = "sha256:444b580fc72fd6887e650acd3e575e18cdc79dbcf42fb4030b491057921f61f8", size = 10738279, upload-time = "2026-05-14T13:44:18.7Z" }, + { url = "https://files.pythonhosted.org/packages/86/ca/caa871ee7be718c45256fada4e16a218ee3e33f0c4a46b729a60a24912e6/ruff-0.15.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6590d009e7cb7ebf36f83dbdd44a3fa48a0994ff6f1cdc1b08006abe58f98dc7", size = 11124798, upload-time = "2026-05-14T13:44:06.427Z" }, + { url = "https://files.pythonhosted.org/packages/d3/19/43f5f2e568dddde567fc41f8471f9432c09563e19d3e617a48cfa52f8f0a/ruff-0.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1c26d2f66163deeb6e08d8b39fbbe983ce3c71cea06a6d7591cfd1421793c629", size = 10460761, upload-time = "2026-05-14T13:44:04.375Z" }, + { url = "https://files.pythonhosted.org/packages/99/df/cf938cd6de3003178f03ad7c1ea2a6c099468c03a35037985070b37e76be/ruff-0.15.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbd6f94b434f896308e4d57fb7bfde0d02b99f7a64b3bdab0fdfa6a864203a5", size = 10804451, upload-time = "2026-05-14T13:44:25.221Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7d/5d0973129b154ded2225729169d7068f26b467760b146493fde138415f23/ruff-0.15.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3259f3be4d181bda591da5db2571aed6853c6a048157756448020bc6c5cd22", size = 10534285, upload-time = "2026-05-14T13:44:08.888Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e3/6b999bbc66cd51e5f073842bc2a3995e99c5e0e72e16b15e7261f7abf57a/ruff-0.15.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae9c17e5eb4430c154e76abc25d79a318190f5a997f38fb6b114416c5319ffc9", size = 11312063, upload-time = "2026-05-14T13:44:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/af/5a/642639e9f5db04f1e97fbd6e091c6fd20725bdf072fb114d00eefb9e6eb8/ruff-0.15.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e2e39bff6c341f4b577a21b801326fab0b11847f48fcaa83f00a113c9b3cb55", size = 12183079, upload-time = "2026-05-14T13:44:01.634Z" }, + { url = "https://files.pythonhosted.org/packages/19/4c/7585735f6b53b0f12de13618b2f7d250a844f018822efc899df2e7b8295f/ruff-0.15.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8d9a8e08013542e94d3220bc5b62cc3e5ef87c5f74bff367d3fac14fab013e6", size = 11440833, upload-time = "2026-05-14T13:43:59.043Z" }, + { url = "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc411dfebe5eebe55ce041c6ae080eb7668955e866daa2fbb16692a784f1c4ca", size = 11434486, upload-time = "2026-05-14T13:44:27.761Z" }, + { url = "https://files.pythonhosted.org/packages/e1/4e/62c9b999875d4f14db80f277c030578f5e249c9852d65b7ac7ad0b43c041/ruff-0.15.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:768494eb08b9cee54e2fd27969966f74db5a57f6eaa7a90fcb3306af34dfc4bd", size = 11385189, upload-time = "2026-05-14T13:44:13.704Z" }, + { url = "https://files.pythonhosted.org/packages/fc/89/7e959047a104df3eb12863447c110140191fc5b6c4f379ea2e803fcdb0e4/ruff-0.15.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fb75f9a3a7e42ffe117d734494e6c5e5cb3565d66e12612cb63d0e572a41a5b6", size = 10781380, upload-time = "2026-05-14T13:43:56.734Z" }, + { url = "https://files.pythonhosted.org/packages/ff/52/5fd18f3b88cab63e88aa11516b3b4e1e5f720e5c330f8dbe5c26210f41f8/ruff-0.15.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8cb74dd33bb2f6613faf7fc03b660053b5ac4f80e706d5788c6335e2a8048d51", size = 10540605, upload-time = "2026-05-14T13:44:20.748Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e0/9e35f338990d3e41a82875ff7053ffe97541dae81c9d02143177f381d572/ruff-0.15.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7ef823f817fcd191dc934e984be9cf4094f808effa16f2542ad8e821ba02bbf2", size = 11036554, upload-time = "2026-05-14T13:44:16.256Z" }, + { url = "https://files.pythonhosted.org/packages/c2/13/070fb048c24080fba188f66371e2a92785be257ad02242066dc7255ac6e9/ruff-0.15.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f345a13937bd7f09f6f5d19fa0721b0c103e00e7f62bc67089a8e5e037719e0b", size = 11528133, upload-time = "2026-05-14T13:44:22.808Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/b1e1666aef7fc6555094d73ae6cd981701781ae85b97ceefc0eebd0b4668/ruff-0.15.13-py3-none-win32.whl", hash = "sha256:4044f94208b3b05ba0fc4a4abd0558cf4d6459bd18325eead7fd8cc66f909b41", size = 10721455, upload-time = "2026-05-14T13:44:35.697Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a6/870a3e8a50590bb92be184ad928c2922f088b00d9dc5c5ec7b924ee08c22/ruff-0.15.13-py3-none-win_amd64.whl", hash = "sha256:7064884d442b7d477b4e7473d12da7f08851d2b1982763c5d3f388a19468a1a4", size = 11900409, upload-time = "2026-05-14T13:44:30.389Z" }, + { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, +] + +[[package]] +name = "spec-validator" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "ruff", specifier = ">=0.6" }, +] From 6add2cc198a09c67296298a7a371886f30800241 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 28 May 2026 01:30:10 +0200 Subject: [PATCH 2/3] chore: declare spec-validator capability + sync capabilities doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capability-sync check from #340 requires every tool with a `**Capability:**` declaration to carry a row in `docs/labels-and-capabilities.md`, and every tool README to declare its capability. `tools/spec-validator/README.md` predates the rule; add the line (matches sibling meta tools — `skill-and-tool-validator`, `spec-loop` — at `capability:setup`) and the corresponding table row. Also fix a stale ref in the README: `tools/skill-validator/` → `tools/skill-and-tool-validator/` (renamed in #340). Generated-by: Claude Code (Opus 4.7) --- docs/labels-and-capabilities.md | 1 + tools/spec-validator/README.md | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/labels-and-capabilities.md b/docs/labels-and-capabilities.md index 346c7ccb..1792142a 100644 --- a/docs/labels-and-capabilities.md +++ b/docs/labels-and-capabilities.md @@ -187,6 +187,7 @@ Tools under [`tools/`](../tools/). Tools with two values (separated by | [`tools/skill-evals`](../tools/skill-evals/) | `capability:setup` + `capability:stats` | Eval harness for skills; the harness is setup infrastructure, the run output is governance evidence | | [`tools/skill-and-tool-validator`](../tools/skill-and-tool-validator/) | `capability:setup` | Skill-frontmatter and convention validator | | [`tools/spec-status-index`](../tools/spec-status-index/) | `capability:setup` + `capability:stats` | Index of spec / RFC implementation status — substrate that also doubles as a governance/stats view | +| [`tools/spec-validator`](../tools/spec-validator/) | `capability:setup` | Spec-frontmatter and body-section validator — counterpart to `skill-and-tool-validator` for `tools/spec-loop/specs/` | | [`tools/vulnogram`](../tools/vulnogram/) | `capability:resolve` | ASF Vulnogram CVE-allocation client | A tool's capabilities are determined by its **use-case lifecycle diff --git a/tools/spec-validator/README.md b/tools/spec-validator/README.md index c94e2915..47822015 100644 --- a/tools/spec-validator/README.md +++ b/tools/spec-validator/README.md @@ -13,8 +13,10 @@ # spec-validator +**Capability:** capability:setup + Validates spec files in `tools/spec-loop/specs/` — the counterpart to -`tools/skill-validator/` for the spec side of the framework. +`tools/skill-and-tool-validator/` for the spec side of the framework. ## What it checks From 072a83d6e371e5839e877ff1b3b16857f99d9863 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 28 May 2026 01:38:39 +0200 Subject: [PATCH 3/3] chore: clear codeql findings on test file (unused imports + var) CodeQL flagged two unused names in `tests/test_spec_validator.py`: - unused imports `REQUIRED_FRONTMATTER_KEYS` and `validate_file` - unused local variable `text` in `test_missing_code_block` (left over from an earlier draft; the test already uses `spec_no_code` for the actual assertion) Remove all three. Ruff clean; 57 tests still pass. Generated-by: Claude Code (Opus 4.7) --- tools/spec-validator/tests/test_spec_validator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/spec-validator/tests/test_spec_validator.py b/tools/spec-validator/tests/test_spec_validator.py index d42caf9d..0e91f1cf 100644 --- a/tools/spec-validator/tests/test_spec_validator.py +++ b/tools/spec-validator/tests/test_spec_validator.py @@ -28,7 +28,6 @@ ALLOWED_KIND, ALLOWED_MODE, ALLOWED_STATUS, - REQUIRED_FRONTMATTER_KEYS, REQUIRED_SECTIONS, extract_section_headings, get_section_body, @@ -37,7 +36,6 @@ parse_frontmatter, run_validation, validate_body, - validate_file, validate_frontmatter, validation_has_code_block, ) @@ -214,8 +212,6 @@ def test_valid_spec_has_code_block(self) -> None: assert validation_has_code_block(_VALID_SPEC) is True def test_missing_code_block(self) -> None: - text = _VALID_SPEC.replace("```bash\n uv run --project tools/example --group dev pytest\n ```", "Run pytest.") - # simpler replacement spec = _make_spec() spec_no_code = spec.replace("```bash\npytest\n```", "Run pytest manually.") assert validation_has_code_block(spec_no_code) is False