From e17ac4d6f3022c7ebf8fd587b00d0ebddff04dc7 Mon Sep 17 00:00:00 2001 From: Justin McLean Date: Wed, 1 Jul 2026 22:17:23 +1000 Subject: [PATCH] feat(validator): add capability taxonomy coverage check (#18, SOFT advisory) Reads the Axis 1 (skill) and Axis 2 (tool) capability vocabulary tables from docs/labels-and-capabilities.md and verifies every taxonomy entry appears in at least one mapping-table row; entries marked *(reserved)* or *(future)* are exempt. Cross-checks the SKILL_CAPABILITIES / TOOL_CAPABILITIES code constants against the parsed vocabulary. The reserved/future marker accepts an elaborated parenthetical (e.g. *(future work)*, *(reserved for #999)*), not only the exact forms. Co-authored-by: Justin McLean --- .../src/skill_and_tool_validator/__init__.py | 192 +++++++++++++++ .../tests/test_validator.py | 227 ++++++++++++++++++ 2 files changed, 419 insertions(+) diff --git a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py index 206efe61..ea4547f5 100644 --- a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py +++ b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py @@ -111,6 +111,15 @@ public branch names must not reveal embargo context. Lines in explicit "bad example" contexts (containing ``**bad**`` or ``bad:``) are exempt. Advisory only. +18. Capability taxonomy coverage (SOFT) — reads the Axis 1 (skill) and + Axis 2 (tool) capability vocabulary tables from + ``docs/labels-and-capabilities.md`` and verifies that every taxonomy + entry appears in at least one row of the corresponding mapping table + (``## Capability to skill map`` / ``## Capability to tool map``). + Vocabulary entries marked ``*(reserved)*`` or ``*(future)*`` in their + Definition column are exempted. Also cross-checks that the hardcoded + ``SKILL_CAPABILITIES`` and ``TOOL_CAPABILITIES`` code constants match + the parsed vocabulary so code and docs stay in sync. Advisory only. SOFT categories surface as advisory warnings (stderr) without failing the run unless ``--strict`` is passed. @@ -472,6 +481,10 @@ def _read_mode_table() -> dict[str, str]: # SOFT advisory: branch name examples in code blocks that contain embargo-breaking # terms (CVE IDs, security, vulnerability, advisory) before public disclosure. BRANCH_CONFIDENTIALITY_CATEGORY = "branch-name-confidentiality" +# SOFT advisory: capability taxonomy vocabulary entry in docs/labels-and-capabilities.md +# has no skill/tool implementation in the mapping tables, or the hardcoded +# SKILL_CAPABILITIES / TOOL_CAPABILITIES constants have drifted from the doc. +CAPABILITY_TAXONOMY_CATEGORY = "capability-taxonomy" # The `magpie-` namespace prefix every installed framework skill carries. SKILL_NAME_PREFIX = "magpie-" @@ -492,6 +505,7 @@ def _read_mode_table() -> dict[str, str]: OVERRIDE_CONTRACT_CATEGORY, TEMPLATE_DRIFT_CATEGORY, BRANCH_CONFIDENTIALITY_CATEGORY, + CAPABILITY_TAXONOMY_CATEGORY, } ) HARD_CATEGORIES: frozenset[str] = frozenset( @@ -2018,6 +2032,181 @@ def validate_capability_sync(root: Path | None = None) -> Iterable[Violation]: ) +# --------------------------------------------------------------------------- +# Capability taxonomy coverage check (aspect #18, SOFT) +# --------------------------------------------------------------------------- + +# Section anchors for the Axis 1 and Axis 2 vocabulary tables. +_AXIS1_ANCHOR = "**Axis 1 — skill capability**" +_AXIS2_ANCHOR = "**Axis 2 — tool capability**" +# Marker for reserved or future taxonomy entries in the Definition column. +# Allows an elaborated parenthetical, e.g. ``*(future work)*`` or +# ``*(reserved for #999)*`` — only the leading keyword is significant. +_RESERVED_FUTURE_RE = re.compile(r"\*\((?:reserved|future)\b[^)]*\)\*", re.IGNORECASE) + + +def _parse_capability_vocabulary_tables( + text: str, +) -> tuple[dict[str, bool], dict[str, bool]]: + """Parse the Axis 1 and Axis 2 vocabulary tables from ``docs/labels-and-capabilities.md``. + + Returns ``(skill_vocab, tool_vocab)`` where each dict maps a capability + token (e.g. ``capability:triage``, ``contract:tracker``) to a bool that + is ``True`` when the entry is marked ``*(reserved)*`` or ``*(future)*`` + in its Definition column and therefore exempt from the coverage check. + """ + + def _extract_vocab(section_text: str) -> dict[str, bool]: + result: dict[str, bool] = {} + for line in section_text.splitlines(): + if not line.startswith("|"): + continue + if "|---" in line: + continue + cells = [c.strip() for c in line.strip("|").split("|")] + if not cells: + continue + first = cells[0] + # Vocabulary table: first cell contains the capability token itself. + m = _CAPABILITY_TOKEN_RE.search(first) + if not m: + continue + token = m.group(1) + rest = " ".join(cells[1:]) + is_exempt = bool(_RESERVED_FUTURE_RE.search(rest)) + result[token] = is_exempt + return result + + axis1_start = text.find(_AXIS1_ANCHOR) + axis2_start = text.find(_AXIS2_ANCHOR) + + if axis1_start == -1 or axis2_start == -1: + return {}, {} + + axis1_section = text[axis1_start:axis2_start] + # Axis 2 ends at the next level-3 heading ("### 3.") or end of document. + axis2_end = re.search(r"\n###\s", text[axis2_start:]) + axis2_section = text[axis2_start : axis2_start + axis2_end.start()] if axis2_end else text[axis2_start:] + + return _extract_vocab(axis1_section), _extract_vocab(axis2_section) + + +def validate_capability_taxonomy_coverage(root: Path | None = None) -> Iterable[Violation]: + """Check that every taxonomy vocabulary entry has at least one implementation. + + Reads the Axis 1 (skill) and Axis 2 (tool) vocabulary tables from + ``docs/labels-and-capabilities.md``, then verifies that each capability + token appears in at least one row of the corresponding mapping table + (``## Capability to skill map`` / ``## Capability to tool map``). + + Entries marked ``*(reserved)*`` or ``*(future)*`` in their Definition + column are exempted from the coverage requirement — they intentionally + have no current implementation. + + Also cross-checks that ``SKILL_CAPABILITIES`` and ``TOOL_CAPABILITIES`` + code constants match the parsed vocabulary so the two stay in sync. + + All violations are SOFT advisories. + """ + repo_root = root or find_repo_root() + doc_path = repo_root / DOCS_LABELS_AND_CAPABILITIES + if not doc_path.exists(): + return + + try: + doc_text = doc_path.read_text(encoding="utf-8") + except OSError: + return + + skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc_text) + if not skill_vocab and not tool_vocab: + yield Violation( + doc_path, + None, + "could not parse Axis 1 / Axis 2 vocabulary tables from " + "docs/labels-and-capabilities.md — check that the section anchors are present", + category=CAPABILITY_TAXONOMY_CATEGORY, + ) + return + + doc_skills = _parse_capability_doc_table(doc_text, _SKILL_TABLE_HEADER) + doc_tools = _parse_capability_doc_table(doc_text, _TOOL_TABLE_HEADER) + + # Collect all capability tokens that appear in the mapping tables. + implemented_skill_caps: set[str] = set() + for caps in doc_skills.values(): + implemented_skill_caps |= caps + implemented_tool_caps: set[str] = set() + for caps in doc_tools.values(): + implemented_tool_caps |= caps + + # Check Axis 1: every non-exempt vocabulary entry must appear in the skill map. + for token, is_exempt in sorted(skill_vocab.items()): + if is_exempt: + continue + if token not in implemented_skill_caps: + yield Violation( + doc_path, + None, + f"capability taxonomy entry '{token}' (Axis 1) has no implementation in " + f"'## Capability to skill map' — add a skill row or mark the entry " + f"*(reserved)* / *(future)* in the vocabulary table", + category=CAPABILITY_TAXONOMY_CATEGORY, + ) + + # Check Axis 2: every non-exempt vocabulary entry must appear in the tool map. + for token, is_exempt in sorted(tool_vocab.items()): + if is_exempt: + continue + if token not in implemented_tool_caps: + yield Violation( + doc_path, + None, + f"capability taxonomy entry '{token}' (Axis 2) has no implementation in " + f"'## Capability to tool map' — add a tool row or mark the entry " + f"*(reserved)* / *(future)* in the vocabulary table", + category=CAPABILITY_TAXONOMY_CATEGORY, + ) + + # Cross-check: SKILL_CAPABILITIES code constant vs parsed vocabulary. + parsed_skill_set = set(skill_vocab.keys()) + if parsed_skill_set != SKILL_CAPABILITIES: + extra_in_code = SKILL_CAPABILITIES - parsed_skill_set + extra_in_doc = parsed_skill_set - SKILL_CAPABILITIES + parts = [] + if extra_in_code: + parts.append(f"in code but not in taxonomy: {sorted(extra_in_code)}") + if extra_in_doc: + parts.append(f"in taxonomy but not in code: {sorted(extra_in_doc)}") + yield Violation( + doc_path, + None, + "SKILL_CAPABILITIES constant has drifted from the Axis 1 vocabulary in " + f"docs/labels-and-capabilities.md — {'; '.join(parts)}; " + "update the constant to match the taxonomy", + category=CAPABILITY_TAXONOMY_CATEGORY, + ) + + # Cross-check: TOOL_CAPABILITIES code constant vs parsed vocabulary. + parsed_tool_set = set(tool_vocab.keys()) + if parsed_tool_set != TOOL_CAPABILITIES: + extra_in_code = TOOL_CAPABILITIES - parsed_tool_set + extra_in_doc = parsed_tool_set - TOOL_CAPABILITIES + parts = [] + if extra_in_code: + parts.append(f"in code but not in taxonomy: {sorted(extra_in_code)}") + if extra_in_doc: + parts.append(f"in taxonomy but not in code: {sorted(extra_in_doc)}") + yield Violation( + doc_path, + None, + "TOOL_CAPABILITIES constant has drifted from the Axis 2 vocabulary in " + f"docs/labels-and-capabilities.md — {'; '.join(parts)}; " + "update the constant to match the taxonomy", + category=CAPABILITY_TAXONOMY_CATEGORY, + ) + + # --------------------------------------------------------------------------- # Lowercase -f field check (Pattern 2) # --------------------------------------------------------------------------- @@ -3270,6 +3459,9 @@ def run_validation(root: Path | None = None) -> list[Violation]: continue violations.extend(validate_branch_name_confidentiality(doc_path, doc_text)) + # Capability taxonomy coverage: every vocab entry has an implementation or is reserved. + violations.extend(validate_capability_taxonomy_coverage(repo_root)) + return violations diff --git a/tools/skill-and-tool-validator/tests/test_validator.py b/tools/skill-and-tool-validator/tests/test_validator.py index 48cdf459..8ad08174 100644 --- a/tools/skill-and-tool-validator/tests/test_validator.py +++ b/tools/skill-and-tool-validator/tests/test_validator.py @@ -35,6 +35,7 @@ ALLOWED_MODES, ASF_COUPLING_CATEGORY, BRANCH_CONFIDENTIALITY_CATEGORY, + CAPABILITY_TAXONOMY_CATEGORY, EVAL_COVERAGE_CATEGORY, FORBIDDEN_PATTERNS, GH_LIST_CATEGORY, @@ -54,11 +55,14 @@ PRINCIPLE_CATEGORY, PRIVACY_CATEGORY, SECURITY_PATTERN_CATEGORY, + SKILL_CAPABILITIES, SKILL_SOURCE_CATEGORY, SOFT_CATEGORIES, STATUS_CATEGORY, TEMPLATE_DRIFT_CATEGORY, + TOOL_CAPABILITIES, TRIGGER_PRESERVATION_CATEGORY, + _parse_capability_vocabulary_tables, _parse_modes_doc, _read_mode_table, collect_doc_files, @@ -85,6 +89,7 @@ validate_asf_coupling, validate_branch_name_confidentiality, validate_capability_sync, + validate_capability_taxonomy_coverage, validate_eval_coverage, validate_frontmatter, validate_gh_list_limit, @@ -3209,6 +3214,228 @@ def test_italic_parens_annotation_is_stripped(self, tmp_path: Path) -> None: assert violations == [], [v.message for v in violations] +# --------------------------------------------------------------------------- +# Capability taxonomy vocabulary parser +# --------------------------------------------------------------------------- + + +class TestParseCapabilityVocabularyTables: + """Tests for _parse_capability_vocabulary_tables.""" + + _AXIS1_HEADER = "**Axis 1 — skill capability**" + _AXIS2_HEADER = "**Axis 2 — tool capability**" + + def _doc(self, axis1_rows: str = "", axis2_rows: str = "") -> str: + return ( + f"### 2. capability\n\n" + f"{self._AXIS1_HEADER} (`capability:*`) — the workflow-lifecycle\n\n" + "| Label | Definition |\n" + "|---|---|\n" + f"{axis1_rows}\n" + f"{self._AXIS2_HEADER} (`contract:*` / `substrate:*`) — the interface:\n\n" + "| Label | Kind | Definition |\n" + "|---|---|---|\n" + f"{axis2_rows}\n" + "\n### 3. kind:* — change type\n" + ) + + def test_parses_skill_capabilities(self) -> None: + doc = self._doc( + axis1_rows=( + "| `capability:triage` | Sweep a queue. |\n| `capability:fix` | Implement a fix. |\n" + ), + ) + skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc) + assert "capability:triage" in skill_vocab + assert "capability:fix" in skill_vocab + assert tool_vocab == {} + + def test_parses_tool_capabilities(self) -> None: + doc = self._doc( + axis2_rows=( + "| `contract:tracker` | contract | Issue backend. |\n" + "| `substrate:sandbox` | substrate | Agent isolation. |\n" + ), + ) + skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc) + assert skill_vocab == {} + assert "contract:tracker" in tool_vocab + assert "substrate:sandbox" in tool_vocab + + def test_reserved_marker_sets_exempt_flag(self) -> None: + doc = self._doc( + axis1_rows="| `capability:future-thing` | Planned. *(future)* |\n", + ) + skill_vocab, _ = _parse_capability_vocabulary_tables(doc) + assert skill_vocab.get("capability:future-thing") is True + + def test_non_exempt_entry_has_false_flag(self) -> None: + doc = self._doc( + axis1_rows="| `capability:triage` | Sweep. |\n", + ) + skill_vocab, _ = _parse_capability_vocabulary_tables(doc) + assert skill_vocab.get("capability:triage") is False + + def test_missing_anchors_returns_empty(self) -> None: + skill_vocab, tool_vocab = _parse_capability_vocabulary_tables("No anchors here.") + assert skill_vocab == {} + assert tool_vocab == {} + + +# --------------------------------------------------------------------------- +# Capability taxonomy coverage check +# --------------------------------------------------------------------------- + + +def _seed_taxonomy_repo( + tmp_path: Path, + *, + axis1_rows: str = "", + axis2_rows: str = "", + mapping_skill_rows: str = "", + mapping_tool_rows: str = "", +) -> Path: + """Build a minimal repo with docs/labels-and-capabilities.md for taxonomy tests.""" + root = tmp_path / "repo" + (root / "docs").mkdir(parents=True) + (root / "skills").mkdir(parents=True) + (root / "tools").mkdir(parents=True) + + doc = ( + "# Labels and capabilities\n\n" + "### 2. capability\n\n" + "**Axis 1 — skill capability** (`capability:*`) — lifecycle phase:\n\n" + "| Label | Definition |\n" + "|---|---|\n" + f"{axis1_rows}\n" + "**Axis 2 — tool capability** (`contract:*` / `substrate:*`) — interface:\n\n" + "| Label | Kind | Definition |\n" + "|---|---|---|\n" + f"{axis2_rows}\n" + "\n### 3. kind:*\n\n" + "## Capability to skill map\n\n" + "| Skill | Capability / capabilities |\n" + "|---|---|\n" + f"{mapping_skill_rows}\n" + "## Capability to tool map\n\n" + "| Tool | Capability / capabilities | Role |\n" + "|---|---|---|\n" + f"{mapping_tool_rows}\n" + ) + (root / "docs" / "labels-and-capabilities.md").write_text(doc) + return root + + +class TestValidateCapabilityTaxonomyCoverage: + """Tests for validate_capability_taxonomy_coverage (check #17 — SOFT).""" + + def test_fully_covered_passes(self, tmp_path: Path) -> None: + root = _seed_taxonomy_repo( + tmp_path, + axis1_rows="| `capability:triage` | Sweep. |\n", + mapping_skill_rows="| `alpha` | `capability:triage` |\n", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + taxonomy_violations = [v for v in violations if v.category == CAPABILITY_TAXONOMY_CATEGORY] + # The only expected advisory is the SKILL_CAPABILITIES constant drift (test repo + # has a one-entry taxonomy while the real constant has ten). Filter to coverage-only. + coverage_violations = [v for v in taxonomy_violations if "has no implementation" in v.message] + assert coverage_violations == [] + + def test_uncovered_axis1_entry_flagged(self, tmp_path: Path) -> None: + root = _seed_taxonomy_repo( + tmp_path, + axis1_rows=("| `capability:triage` | Sweep. |\n| `capability:orphan` | Orphaned. |\n"), + mapping_skill_rows="| `alpha` | `capability:triage` |\n", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + assert any("capability:orphan" in v.message and "Axis 1" in v.message for v in violations) + + def test_uncovered_axis2_entry_flagged(self, tmp_path: Path) -> None: + root = _seed_taxonomy_repo( + tmp_path, + axis2_rows="| `contract:orphan-tool` | contract | Orphaned. |\n", + mapping_tool_rows="", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + assert any("contract:orphan-tool" in v.message and "Axis 2" in v.message for v in violations) + + def test_reserved_entry_is_exempt(self, tmp_path: Path) -> None: + root = _seed_taxonomy_repo( + tmp_path, + axis1_rows="| `capability:future-thing` | Planned. *(reserved)* |\n", + mapping_skill_rows="", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + coverage_violations = [ + v + for v in violations + if v.category == CAPABILITY_TAXONOMY_CATEGORY and "has no implementation" in v.message + ] + assert coverage_violations == [] + + def test_future_marker_also_exempt(self, tmp_path: Path) -> None: + root = _seed_taxonomy_repo( + tmp_path, + axis1_rows="| `capability:next-gen` | Next gen. *(future)* |\n", + mapping_skill_rows="", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + coverage_violations = [ + v + for v in violations + if v.category == CAPABILITY_TAXONOMY_CATEGORY and "has no implementation" in v.message + ] + assert coverage_violations == [] + + def test_code_constant_drift_flagged(self, tmp_path: Path) -> None: + # Axis 1 vocabulary has a token not in SKILL_CAPABILITIES. + root = _seed_taxonomy_repo( + tmp_path, + axis1_rows="| `capability:not-in-code` | New. |\n", + mapping_skill_rows="| `alpha` | `capability:not-in-code` |\n", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + assert any("SKILL_CAPABILITIES constant has drifted" in v.message for v in violations) + + def test_tool_constant_drift_flagged(self, tmp_path: Path) -> None: + root = _seed_taxonomy_repo( + tmp_path, + axis2_rows="| `contract:new-adapter` | contract | New. |\n", + mapping_tool_rows="| [`tools/new-adapter`](../tools/new-adapter/) | `contract:new-adapter` | role |\n", + ) + violations = list(validate_capability_taxonomy_coverage(root)) + assert any("TOOL_CAPABILITIES constant has drifted" in v.message for v in violations) + + def test_category_is_soft(self) -> None: + assert CAPABILITY_TAXONOMY_CATEGORY in SOFT_CATEGORIES + + def test_real_repo_passes_clean(self) -> None: + """Taxonomy coverage check must pass against the live repo with no violations.""" + violations = list(validate_capability_taxonomy_coverage()) + assert violations == [], [v.message for v in violations] + + def test_vocabulary_constants_match_live_taxonomy(self) -> None: + """SKILL_CAPABILITIES and TOOL_CAPABILITIES must match the live taxonomy doc.""" + from pathlib import Path + + doc_path = Path("docs/labels-and-capabilities.md") + if not doc_path.exists(): + pytest.skip("labels-and-capabilities.md not found — not in repo root") + doc_text = doc_path.read_text(encoding="utf-8") + skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc_text) + assert set(skill_vocab.keys()) == SKILL_CAPABILITIES, ( + f"SKILL_CAPABILITIES constant is out of sync with taxonomy. " + f"In code only: {SKILL_CAPABILITIES - set(skill_vocab.keys())}; " + f"In doc only: {set(skill_vocab.keys()) - SKILL_CAPABILITIES}" + ) + assert set(tool_vocab.keys()) == TOOL_CAPABILITIES, ( + f"TOOL_CAPABILITIES constant is out of sync with taxonomy. " + f"In code only: {TOOL_CAPABILITIES - set(tool_vocab.keys())}; " + f"In doc only: {set(tool_vocab.keys()) - TOOL_CAPABILITIES}" + ) + + # --------------------------------------------------------------------------- # Eval-coverage check # ---------------------------------------------------------------------------