diff --git a/docs/labels-and-capabilities.md b/docs/labels-and-capabilities.md index 76a3ecff..db2d4c73 100644 --- a/docs/labels-and-capabilities.md +++ b/docs/labels-and-capabilities.md @@ -245,7 +245,7 @@ it implements multiple contracts (e.g. `tools/gmail` provides both | [`tools/jira`](../tools/jira/) | `contract:tracker` | JIRA REST substrate (read-only today; write subcommands tracked in [#301](https://github.com/apache/magpie/issues/301)) | | [`tools/mail-archive`](../tools/mail-archive/) | `contract:mail-archive` | Adapter contract for public mail-archive backends (PonyMail, Hyperkitty, Discourse, Google Groups, GitHub Discussions). Pure interface spec. | | [`tools/mail-source`](../tools/mail-source/) | `contract:mail-source` | Mail-source backend abstraction (mbox / IMAP / Mailman 3) feeding a uniform inbound thread/message view to the intake pipeline | -| [`tools/ponymail`](../tools/ponymail/) | `contract:mail-archive` | PonyMail public mail-archive substrate (ASF `lists.apache.org`); implements the `tools/mail-archive/` contract | +| [`tools/ponymail`](../tools/ponymail/) | `contract:mail-archive` + `contract:mail-source` | PonyMail public mail-archive substrate (ASF `lists.apache.org`); implements the `tools/mail-archive/` contract for archive reads and the `tools/mail-source/` contract for inbound list-traffic ingestion | | [`tools/scan-format`](../tools/scan-format/) | `contract:scan-format` | Adapter contract for security-scanner report formats (ASVS reference); reads a scan's finding index + per-finding evidence for the `security-issue-import-from-scan` pipeline. | | [`tools/permission-audit`](../tools/permission-audit/) | `substrate:sandbox` | Audit + atomically edit Claude Code `permissions.allow[]` entries; backs `/magpie-setup verify --apply-permission-audit` (check 8d) | | [`tools/pr-management-stats`](../tools/pr-management-stats/) | `substrate:analytics` | PR-backlog analytics engine | @@ -258,6 +258,7 @@ it implements multiple contracts (e.g. `tools/gmail` provides both | [`tools/skill-evals`](../tools/skill-evals/) | `substrate:framework-dev` | Eval harness for skills; framework-dev infrastructure whose run output is governance evidence | | [`tools/skill-and-tool-validator`](../tools/skill-and-tool-validator/) | `substrate:framework-dev` | Skill-frontmatter and convention validator | | [`tools/spec-status-index`](../tools/spec-status-index/) | `substrate:framework-dev` + `substrate:analytics` | Index of spec / RFC implementation status — framework-dev substrate that also doubles as a governance/stats view (`analytics`) | +| [`tools/vendor-neutrality-score`](../tools/vendor-neutrality-score/) | `substrate:framework-dev` + `substrate:analytics` | Deterministic vendor-neutrality score — reads each contract tool's `**Kind:**` / `**Vendor:**` metadata and scores per-contract + per-skill neutrality (`analytics`); backs the score block in [`docs/vendor-neutrality.md`](vendor-neutrality.md) | | [`tools/spec-validator`](../tools/spec-validator/) | `substrate:framework-dev` | Spec-frontmatter and body-section validator — counterpart to `skill-and-tool-validator` for `tools/spec-loop/specs/` | | [`tools/symlink-lint`](../tools/symlink-lint/) | `substrate:framework-dev` | Self-adoption symlink hygiene — rejects cyclic symlinks and misdirected skill relays (canonical/relay target-correctness) | | [`tools/pilot-report-validator`](../tools/pilot-report-validator/) | `substrate:framework-dev` | Adopter pilot-report validator — required frontmatter keys, no unfilled placeholders, valid profile, and required body sections; counterpart to `spec-validator` for `docs/pilot-report-template.md` | @@ -286,7 +287,7 @@ backend the adopter wired in. The framework consumes four: |---|---|---|---|---| | GitHub MCP | `mcp__github__*` | [`tools/github`](../tools/github/) | `contract:tracker` + `contract:source-control` | — | | Gmail MCP (claude.ai) | `mcp__claude_ai_Gmail__*` | [`tools/gmail`](../tools/gmail/) | `contract:mail-source` + `contract:mail-draft` + `contract:mail-archive` | — | -| PonyMail MCP (`apache/comdev`) | `mcp__ponymail__*` | [`tools/ponymail`](../tools/ponymail/) | `contract:mail-archive` | ASF | +| PonyMail MCP (`apache/comdev`) | `mcp__ponymail__*` | [`tools/ponymail`](../tools/ponymail/) | `contract:mail-archive` + `contract:mail-source` | ASF | | apache-projects MCP (`apache/comdev`) | `mcp__apache-projects__*` | [`tools/apache-projects`](../tools/apache-projects/) | `contract:project-metadata` | ASF | Non-MCP backends fulfil the same contracts: JIRA is reached over REST diff --git a/docs/vendor-neutrality.md b/docs/vendor-neutrality.md index 10cd69ba..c69c9405 100644 --- a/docs/vendor-neutrality.md +++ b/docs/vendor-neutrality.md @@ -22,6 +22,9 @@ - [What keeps it neutral over time](#what-keeps-it-neutral-over-time) - [The contribution model — neutrality as an invitation](#the-contribution-model--neutrality-as-an-invitation) - [Status at a glance](#status-at-a-glance) + - [Vendor-neutrality score](#vendor-neutrality-score) + - [How the score is computed](#how-the-score-is-computed) + - [What the number means](#what-the-number-means) - [What "vendor neutral" does and does not claim](#what-vendor-neutral-does-and-does-not-claim) - [See also](#see-also) @@ -490,6 +493,89 @@ coverage without pretending one team can implement an open-ended set. adding a backend is an adapter against a documented contract, not a change to any skill. +## Vendor-neutrality score + +The six axes above are the *narrative*. This section is the +*measurement* — a deterministic score computed straight from repository +metadata by +[`tools/vendor-neutrality-score`](../tools/vendor-neutrality-score/), so +the number is reproducible from the source tree and cannot quietly drift +from the code. + +### How the score is computed + +Neutrality is measured per **capability contract** — the `contract:*` +verbs a skill depends on. Substrate tools (Magpie's own machinery: +sandboxing, analytics, framework-dev) are excluded, because they are not +a vendor choice. + +Every contract tool declares three fields in its README: `**Capability:**` +(the contract it fulfils), `**Kind:**` (`interface` for a pure spec, +`implementation` for a concrete backend), and `**Vendor:**` (the backend +identity). The scorer reads them and applies one rule per contract +**class**: + +- **vendor-backed** → GREEN once **two or more distinct backend vendors** + implement it. One backend, however good, is a *default*, not + neutrality. Interface specs do not count — only shipping backends do. +- **agnostic** → GREEN by construction: a single vendor-neutral spec + serves every backend, so there is no vendor to be neutral *between*. +- **single-organisation** → GREEN by exemption: the capability is bound + to one organisation's data model (e.g. ASF governance rosters); there + is no vendor choice to make. + +The overall score is `green contracts / total contracts` — a hard, +falsifiable number. Add a second outbound-mail backend and `mail-draft` +flips to green on the next run; remove a backend and its contract flips +back. The same rule then classifies every **skill**: *capability-pure* +if it names no backend, *portable* if every backend it invokes has an +alternative, and *vendor-coupled* only if it reaches for the sole +implementation of a capability. + + +**Overall vendor-neutrality score: 8/9 capability contracts (89%).** Generated by [`tools/vendor-neutrality-score`](../tools/vendor-neutrality-score/); re-run it to refresh this section. + +| Capability contract | Neutral? | Class | Backends today | Basis | +|---|---|---|---|---| +| `contract:tracker` | ✅ | vendor-backed | Atlassian, GitHub | 2 backend vendors: Atlassian, GitHub | +| `contract:source-control` | ✅ | vendor-backed | Git, GitHub, Subversion | 3 backend vendors: Git, GitHub, Subversion | +| `contract:mail-archive` | ✅ | vendor-backed | Google, PonyMail | 2 backend vendors: Google, PonyMail | +| `contract:mail-source` | ✅ | vendor-backed | Google, PonyMail | 2 backend vendors: Google, PonyMail | +| `contract:mail-draft` | ❌ | vendor-backed | Google | only 1 backend vendor (Google); needs 1 more | +| `contract:cve-authority` | ✅ | vendor-backed | CVE.org, Vulnogram | 2 backend vendors: CVE.org, Vulnogram | +| `contract:report-relay` | ✅ | agnostic | — | vendor-neutral by construction — one spec serves every backend | +| `contract:scan-format` | ✅ | agnostic | — | vendor-neutral by construction — one spec serves every backend | +| `contract:project-metadata` | ✅ | single-org | ASF | single-organisation capability (ASF); no vendor choice to make | + +**Per-skill assessment: 59/63 skills carry no vendor lock-in.** A skill is *capability-pure* when it names no backend at all, *portable* when every backend it names has an alternative (its contract is green), and *vendor-coupled* only when it reaches for a backend that is the sole implementation of a capability. + +| Skill neutrality | Count | +|---|---| +| capability-pure (names no backend) | 9 | +| portable (named backends are swappable) | 50 | +| vendor-coupled (sole-backend dependency) | 4 | + +Organization scope (declared, orthogonal to vendor): ASF = 14, agnostic = 49. + +Vendor-coupled skills (the only lock-ins today): + +- `security-issue-import` — `Google` → `contract:mail-draft` +- `security-issue-import-via-forwarder` — `Google` → `contract:mail-draft` +- `security-issue-invalidate` — `Google` → `contract:mail-draft` +- `security-issue-sync` — `Google` → `contract:mail-draft` + + +### What the number means + +89% is not "89% done." It reads as: **eight of nine capabilities already +work across more than one vendor, and the ninth — outbound mail drafting +— is one adapter away.** The architecture privileges no vendor on any of +the nine axes; the single red cell is a missing *implementation* (a +second `mail-draft` backend), tracked in the open, not a design that +assumes Gmail. Likewise only four skills touch that one lock-in, and each +does so through the `mail-draft` contract — swap in a second backend and +all four become portable without a line of skill code changing. + ## What "vendor neutral" does and does not claim To keep the marketing honest and the engineering claim precise: diff --git a/pyproject.toml b/pyproject.toml index bb2dd8dd..1eb11333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,4 +114,5 @@ members = [ "tools/spec-validator", "tools/symlink-lint", "tools/vcs", + "tools/vendor-neutrality-score", ] diff --git a/tools/apache-projects/README.md b/tools/apache-projects/README.md index 4f70f2a8..4981d084 100644 --- a/tools/apache-projects/README.md +++ b/tools/apache-projects/README.md @@ -14,6 +14,10 @@ **Capability:** contract:project-metadata +**Kind:** implementation + +**Vendor:** ASF + **Organization:** ASF ASF project-metadata substrate. Read-only, unauthenticated client diff --git a/tools/asf-svn/README.md b/tools/asf-svn/README.md index 2f8239f5..6ca3a764 100644 --- a/tools/asf-svn/README.md +++ b/tools/asf-svn/README.md @@ -15,6 +15,10 @@ **Capability:** contract:source-control +**Kind:** implementation + +**Vendor:** Subversion + **Organization:** ASF ASF SVN tool adapter — the Subversion counterpart to diff --git a/tools/cve-org/README.md b/tools/cve-org/README.md index 2306a768..34a1121c 100644 --- a/tools/cve-org/README.md +++ b/tools/cve-org/README.md @@ -14,6 +14,10 @@ **Capability:** contract:cve-authority +**Kind:** implementation + +**Vendor:** CVE.org + CVE.org publication client. Submits CVE records via the CVE.org REST API; consumed by `security-cve-allocate` once a CVE has been allocated via the ASF Vulnogram path. See [`tool.md`](tool.md) for the protocol detail and `cve.org` field mapping. ## Prerequisites diff --git a/tools/cve-tool-vulnogram/README.md b/tools/cve-tool-vulnogram/README.md index 417e7a7a..36f15457 100644 --- a/tools/cve-tool-vulnogram/README.md +++ b/tools/cve-tool-vulnogram/README.md @@ -14,6 +14,10 @@ **Capability:** contract:cve-authority +**Kind:** implementation + +**Vendor:** Vulnogram + **Organization:** ASF ASF Vulnogram CVE-allocation client. OAuth-authenticated API that allocates a CVE ID through the ASF's Vulnogram instance and publishes the CVE record. Consumed by `security-cve-allocate`. See [`allocation.md`](allocation.md), [`record.md`](record.md), and [`bot-credits-policy.md`](bot-credits-policy.md) for the protocol. diff --git a/tools/cve-tool/README.md b/tools/cve-tool/README.md index d72ae50f..e51837ea 100644 --- a/tools/cve-tool/README.md +++ b/tools/cve-tool/README.md @@ -26,6 +26,10 @@ **Capability:** contract:cve-authority +**Kind:** interface + +**Vendor:** agnostic + ## Prerequisites - **Runtime:** None — this directory is a Markdown contract spec; no executable code ships here. It is read by the framework, not run. diff --git a/tools/forwarder-relay/README.md b/tools/forwarder-relay/README.md index 2c6e574c..ef1e98ed 100644 --- a/tools/forwarder-relay/README.md +++ b/tools/forwarder-relay/README.md @@ -30,6 +30,10 @@ **Capability:** contract:report-relay +**Kind:** interface + +**Vendor:** agnostic + A forwarder-relay adapter is a pluggable seam that teaches the security skills how to recognise an inbound report that arrived **through a relay** (someone else forwarded the original diff --git a/tools/github-body-field/README.md b/tools/github-body-field/README.md index 898f6f9a..ca4d0322 100644 --- a/tools/github-body-field/README.md +++ b/tools/github-body-field/README.md @@ -21,6 +21,10 @@ **Capability:** contract:tracker +**Kind:** implementation + +**Vendor:** GitHub + Read or rewrite a single `### Field` section of a GitHub issue body **without bringing the body into agent context**. diff --git a/tools/github-rollup/README.md b/tools/github-rollup/README.md index e417cf79..8cdbde22 100644 --- a/tools/github-rollup/README.md +++ b/tools/github-rollup/README.md @@ -20,6 +20,10 @@ **Capability:** contract:tracker +**Kind:** implementation + +**Vendor:** GitHub + Append to (or create) the status-rollup comment on a GitHub issue **without bringing the rollup body into agent context**. diff --git a/tools/github/README.md b/tools/github/README.md index 33f7b1ae..c2ebc069 100644 --- a/tools/github/README.md +++ b/tools/github/README.md @@ -14,6 +14,10 @@ **Capability:** contract:tracker + contract:source-control +**Kind:** implementation + +**Vendor:** GitHub + GitHub REST + GraphQL substrate. Pure read/write wrapper used by every lifecycle phase (triage / intake / fix / resolve / stats). See [`tool.md`](tool.md) for the operation catalogue and the per-area files ([`issue-template.md`](issue-template.md), [`labels.md`](labels.md), [`operations.md`](operations.md), [`project-board.md`](project-board.md), [`status-rollup.md`](status-rollup.md)) for specifics. ## Prerequisites diff --git a/tools/gmail/README.md b/tools/gmail/README.md index 4abf223b..7ed63670 100644 --- a/tools/gmail/README.md +++ b/tools/gmail/README.md @@ -14,6 +14,10 @@ **Capability:** contract:mail-source + contract:mail-draft + contract:mail-archive +**Kind:** implementation + +**Vendor:** Google + Gmail API substrate. Read + draft-only — never sends. Provides two contracts: `mail-source` for inbound report intake (search / read a uniform thread/message view) and `mail-draft` for outbound courtesy-reply diff --git a/tools/jira/README.md b/tools/jira/README.md index 1eebba88..2687813c 100644 --- a/tools/jira/README.md +++ b/tools/jira/README.md @@ -32,6 +32,10 @@ **Capability:** contract:tracker +**Kind:** implementation + +**Vendor:** Atlassian + JIRA REST helpers for the `issue-*` skill family. Adopters with JIRA-based issue trackers wire this in as their tracker bridge; adopters using GitHub Issues or other trackers diff --git a/tools/mail-archive/README.md b/tools/mail-archive/README.md index a1710ed5..45f54469 100644 --- a/tools/mail-archive/README.md +++ b/tools/mail-archive/README.md @@ -24,6 +24,10 @@ **Capability:** contract:mail-archive +**Kind:** interface + +**Vendor:** agnostic + This file defines the adapter contract for **public mail-archive backends** — the seam that lets adopting projects plug a non-ASF archive system (Hyperkitty, Discourse, Google Groups, GitHub diff --git a/tools/mail-source/README.md b/tools/mail-source/README.md index e702628b..aaae458d 100644 --- a/tools/mail-source/README.md +++ b/tools/mail-source/README.md @@ -14,6 +14,10 @@ **Capability:** contract:mail-source +**Kind:** interface + +**Vendor:** agnostic + Mail-source backend abstraction. Pluggable backends (mbox, IMAP, the Gmail API via [`tools/gmail`](../gmail/), future Mailman 3 / Hyperkitty) that feed the security-issue-import intake pipeline a uniform thread/message view. See [`contract.md`](contract.md) for the backend interface. ## Prerequisites diff --git a/tools/ponymail/README.md b/tools/ponymail/README.md index 3a6e7541..51ed4bc7 100644 --- a/tools/ponymail/README.md +++ b/tools/ponymail/README.md @@ -12,7 +12,11 @@ # `tools/ponymail/` -**Capability:** contract:mail-archive +**Capability:** contract:mail-archive + contract:mail-source + +**Kind:** implementation + +**Vendor:** PonyMail **Organization:** ASF diff --git a/tools/scan-format/README.md b/tools/scan-format/README.md index 632e0492..47f0c91d 100644 --- a/tools/scan-format/README.md +++ b/tools/scan-format/README.md @@ -23,6 +23,10 @@ **Capability:** contract:scan-format +**Kind:** interface + +**Vendor:** agnostic + A **scan-format adapter** teaches [`security-issue-import-from-scan`](../../skills/security-issue-import-from-scan/SKILL.md) how to read one security scanner's report layout. The skill is diff --git a/tools/vcs/README.md b/tools/vcs/README.md index 79bc5432..e00ed5a3 100644 --- a/tools/vcs/README.md +++ b/tools/vcs/README.md @@ -18,6 +18,10 @@ **Capability:** contract:source-control +**Kind:** implementation + +**Vendor:** Git + Runnable implementation of the **source-control (VCS) capability** documented in [`tools/github/source-control.md`](../github/source-control.md). It diff --git a/tools/vendor-neutrality-score/README.md b/tools/vendor-neutrality-score/README.md new file mode 100644 index 00000000..3b71caea --- /dev/null +++ b/tools/vendor-neutrality-score/README.md @@ -0,0 +1,90 @@ + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [vendor-neutrality-score](#vendor-neutrality-score) + - [Prerequisites](#prerequisites) + - [What it reads](#what-it-reads) + - [The scoring rule](#the-scoring-rule) + - [Usage](#usage) + - [Run tests](#run-tests) + + + + + +# vendor-neutrality-score + +**Capability:** substrate:framework-dev + substrate:analytics + +A deterministic `uv` tool that scores Magpie's vendor neutrality from +repository metadata alone — no network, no judgement at runtime. It +answers the question raised in +[apache/magpie-site#17](https://github.com/apache/magpie-site/issues/17): +*for each capability contract, does Magpie already work across more than +one vendor, and is any skill locked to a vendor with no alternative?* + +## Prerequisites + +- **Runtime:** Python 3.11+ run via `uv`; stdlib-only (no runtime + dependencies). The `dev` group pulls `pytest`. +- **CLIs:** None beyond the runtime. +- **Credentials / auth:** None. +- **Network:** Runs fully offline; reads `tools/*/README.md` and + `skills/*/SKILL.md` from the local checkout. + +## What it reads + +Three machine-readable inputs, so the same tree always yields the same +score: + +1. **`tools/*/README.md`** — each contract tool declares `**Capability:**` + (the `contract:` it fulfils), `**Kind:**` (`interface` for a pure + spec, `implementation` for a concrete backend), and `**Vendor:**` (the + backend identity, or `agnostic` for an interface). +2. **`skills/*/SKILL.md`** — the `organization:` frontmatter field plus the + skill body, scanned for the concrete backends it names. +3. **The policy** in `src/vendor_neutrality_score/__init__.py` + (`CONTRACT_POLICY`) — which of three neutrality *classes* each contract + belongs to. This is the only hand-maintained input. + +## The scoring rule + +Substrate tools are Magpie's own machinery and never count. Each +capability **contract** is scored by its class: + +| Class | GREEN when | Examples | +|---|---|---| +| `vendor-backed` | ≥ 2 distinct backend vendors implement it | tracker (GitHub + Jira), source-control (Git + Subversion) | +| `agnostic` | always — one vendor-neutral spec serves every backend | report-relay, scan-format | +| `single-org` | always — bound to one organisation's data model; no vendor choice | project-metadata | + +Overall score = `green_contracts / total_contracts`. + +Each **skill** is then classified as *capability-pure* (names no +backend), *portable* (every backend it names has an alternative), or +*vendor-coupled* (reaches for the sole implementation of a capability). +Declared `organization:` scope is reported as an orthogonal dimension. + +## Usage + +```bash +# Human-readable report (per-contract + per-skill summary) +uv run --project tools/vendor-neutrality-score vendor-neutrality-score + +# Machine-readable JSON +uv run --project tools/vendor-neutrality-score vendor-neutrality-score --json + +# Regenerate the block embedded in docs/vendor-neutrality.md +uv run --project tools/vendor-neutrality-score vendor-neutrality-score --markdown + +# CI gate: fail if neutrality drops below a threshold +uv run --project tools/vendor-neutrality-score vendor-neutrality-score --fail-under 80 +``` + +## Run tests + +```bash +uv run --project tools/vendor-neutrality-score --group dev pytest +``` diff --git a/tools/vendor-neutrality-score/pyproject.toml b/tools/vendor-neutrality-score/pyproject.toml new file mode 100644 index 00000000..b0e65d17 --- /dev/null +++ b/tools/vendor-neutrality-score/pyproject.toml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "vendor-neutrality-score" +version = "0.1.0" +description = "Deterministically score Magpie's vendor neutrality per capability contract and per skill." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Apache-2.0" } +dependencies = [] + +[project.scripts] +vendor-neutrality-score = "vendor_neutrality_score:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/vendor_neutrality_score"] + +[tool.ruff] +line-length = 110 +target-version = "py311" +src = ["src", "tests"] + +[tool.ruff.lint] +select = [ + "E", + "W", + "F", + "I", + "B", + "UP", + "SIM", + "C4", + "RUF", +] +ignore = [ + "E501", +] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["B", "SIM"] + +[tool.mypy] +python_version = "3.11" +files = ["src", "tests"] +warn_unused_ignores = true +warn_redundant_casts = true +warn_unreachable = true +check_untyped_defs = true +no_implicit_optional = true +disallow_untyped_defs = true +disallow_incomplete_defs = true + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[tool.pytest.ini_options] +minversion = "8.0" +addopts = "-ra -q" +testpaths = ["tests"] diff --git a/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py b/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py new file mode 100644 index 00000000..0703741e --- /dev/null +++ b/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py @@ -0,0 +1,497 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Deterministic vendor-neutrality score for Magpie. + +The score answers a single question with no hidden judgement: **for each +capability *contract*, does Magpie already work across more than one +vendor — and is any skill locked to a vendor that has no alternative?** + +It reads three machine-readable inputs from the repository and nothing +else, so the same tree always yields the same score: + +1. ``tools/*/README.md`` — every contract tool declares + ``**Capability:**`` (the ``contract:`` it fulfils), + ``**Kind:**`` (``interface`` for a pure spec, ``implementation`` + for a concrete backend), and ``**Vendor:**`` (the backend identity, + or ``agnostic`` for an interface). +2. ``skills/*/SKILL.md`` — the ``organization:`` frontmatter field + (declared org scope) plus the skill body (scanned for the concrete + backends it names). +3. The policy below — which of three neutrality *classes* each + contract belongs to. This is the only hand-maintained input and it + lives in one reviewable place. + +Scoring rule (substrates are Magpie's own machinery and never count): + +* ``vendor-backed`` contract -> GREEN when at least + ``MIN_VENDORS`` distinct backend vendors implement it. +* ``agnostic`` contract -> GREEN by construction (one vendor-neutral + spec serves every backend; there is no vendor to be neutral between). +* ``single-org`` contract -> GREEN by exemption (the capability is + bound to one organisation's data model; there is no vendor choice to + make). + +The overall score is ``green_contracts / total_contracts``. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path + +# --------------------------------------------------------------------------- +# Policy (the only hand-maintained input) +# --------------------------------------------------------------------------- + +VENDOR_BACKED = "vendor-backed" +AGNOSTIC = "agnostic" +SINGLE_ORG = "single-org" + +# Minimum distinct backend vendors a vendor-backed contract needs to be +# considered vendor neutral (the criterion from apache/magpie-site#17). +MIN_VENDORS = 2 + +# contract -> (neutrality class, human-readable capability summary) +CONTRACT_POLICY: dict[str, tuple[str, str]] = { + "contract:tracker": (VENDOR_BACKED, "Issue / PR / board / label backend"), + "contract:source-control": (VENDOR_BACKED, "Branch / commit / diff / push (VCS)"), + "contract:mail-archive": (VENDOR_BACKED, "Mailing-list / forum archive reads"), + "contract:mail-source": (VENDOR_BACKED, "Inbound-mail ingestion (mbox / IMAP / …)"), + "contract:mail-draft": (VENDOR_BACKED, "Outbound mail composition (draft, never send)"), + "contract:cve-authority": (VENDOR_BACKED, "CVE allocation / record management / publication"), + "contract:report-relay": (AGNOSTIC, "Inbound security-report relay detection"), + "contract:scan-format": (AGNOSTIC, "Security-scanner report parsing"), + "contract:project-metadata": (SINGLE_ORG, "Governance rosters / people / releases"), +} + +# Which capability *contract* a skill actually invokes, keyed by +# high-confidence usage signals: MCP tool-call names, specific CLI verbs, +# and canonical hostnames. A skill is coupled only when a contract it +# invokes has no alternative backend — so this maps *usage*, not prose +# mentions (a skill that merely names "GitHub" in a sentence is not caught; +# one that calls ``gh pr`` or ``mcp__github__*`` is). Contracts that are +# agnostic by construction (report-relay, scan-format) and mail-source +# (indistinguishable from archive reads at the token level) are omitted — +# they never change a skill's verdict. +CONTRACT_USAGE_TOKENS: dict[str, tuple[str, ...]] = { + "contract:tracker": ( + r"mcp__github__", + r"\bgh (?:pr|issue|api|search|run|workflow|release|label)\b", + r"\bJIRA\b", + ), + "contract:source-control": ( + r"\bgit (?:commit|push|checkout|branch|rebase|merge|switch|worktree)\b", + r"\bsvn (?:checkout|commit|update|cat|list|mkdir|import|move|delete|switch|copy)\b", + ), + "contract:mail-archive": ( + r"mcp__ponymail__", + r"mcp__claude_ai_Gmail__(?:search_threads|get_thread|list_)", + ), + "contract:mail-draft": ( + r"mcp__claude_ai_Gmail__create_draft", + r"\bcreate_draft\b", + ), + "contract:cve-authority": ( + r"\bVulnogram\b", + r"\bcveawg\b", + r"\bcve\.org\b", + ), + "contract:project-metadata": ( + r"mcp__apache-projects__", + r"\bprojects\.apache\.org\b", + ), +} + +_CAP_RE = re.compile(r"^\*\*Capability:\*\*[ \t]+(.+)$", re.MULTILINE) +_KIND_RE = re.compile(r"^\*\*Kind:\*\*[ \t]+(.+?)[ \t]*$", re.MULTILINE) +_VENDOR_RE = re.compile(r"^\*\*Vendor:\*\*[ \t]+(.+?)[ \t]*$", re.MULTILINE) +_ORG_RE = re.compile(r"^organization:[ \t]*(.+?)[ \t]*$", re.MULTILINE) + +INTERFACE = "interface" +IMPLEMENTATION = "implementation" + + +# --------------------------------------------------------------------------- +# Data model +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ToolMeta: + """A contract tool's declared vendor-neutrality metadata.""" + + name: str + contracts: tuple[str, ...] + kind: str + vendor: str + + +@dataclass +class ContractResult: + contract: str + klass: str + summary: str + green: bool + basis: str + interfaces: list[str] = field(default_factory=list) + implementations: list[ToolMeta] = field(default_factory=list) + + @property + def vendors(self) -> list[str]: + return sorted({t.vendor for t in self.implementations}) + + +@dataclass +class SkillResult: + name: str + organization: str + contracts: list[str] # capability contracts the skill invokes + verdict: str + coupled: list[tuple[str, str]] # (sole vendor, contract) with no alternative + + +# --------------------------------------------------------------------------- +# Repo discovery + parsing +# --------------------------------------------------------------------------- + + +def find_repo_root(start: Path | None = None) -> Path: + """Walk upward until a directory holds the framework's tool + skill trees.""" + here = (start or Path.cwd()).resolve() + for candidate in (here, *here.parents): + if (candidate / "tools").is_dir() and (candidate / "docs" / "labels-and-capabilities.md").is_file(): + return candidate + # Fall back to the repo this file ships in. + return Path(__file__).resolve().parents[4] + + +def _parse_capabilities(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split("+") if part.strip()) + + +def load_tools(repo_root: Path) -> list[ToolMeta]: + """Read every ``tools/*/README.md`` that declares a ``contract:*`` capability.""" + tools: list[ToolMeta] = [] + for readme in sorted((repo_root / "tools").glob("*/README.md")): + text = readme.read_text(encoding="utf-8") + cap_m = _CAP_RE.search(text) + if not cap_m: + continue + caps = _parse_capabilities(cap_m.group(1)) + contracts = tuple(c for c in caps if c.startswith("contract:")) + if not contracts: + continue # substrate-only tool: Magpie's own machinery, excluded + name = readme.parent.name + kind_m = _KIND_RE.search(text) + vendor_m = _VENDOR_RE.search(text) + if not kind_m or not vendor_m: + raise ValueError( + f"tools/{name}/README.md declares {contracts} but is missing " + f"a '**Kind:**' and/or '**Vendor:**' field required for scoring" + ) + kind = kind_m.group(1).strip() + if kind not in (INTERFACE, IMPLEMENTATION): + raise ValueError( + f"tools/{name}: **Kind:** must be '{INTERFACE}' or '{IMPLEMENTATION}', got '{kind}'" + ) + tools.append(ToolMeta(name=name, contracts=contracts, kind=kind, vendor=vendor_m.group(1).strip())) + return tools + + +def _split_frontmatter(text: str) -> tuple[str, str]: + """Return (frontmatter, body) for a ``---``-delimited markdown file.""" + if text.startswith("---"): + parts = text.split("---", 2) + if len(parts) == 3: + return parts[1], parts[2] + return "", text + + +def load_skills(repo_root: Path) -> list[tuple[str, str, str]]: + """Return (name, organization, body) for every ``skills/*/SKILL.md``.""" + out: list[tuple[str, str, str]] = [] + for skill_md in sorted((repo_root / "skills").glob("*/SKILL.md")): + text = skill_md.read_text(encoding="utf-8") + front, body = _split_frontmatter(text) + org_m = _ORG_RE.search(front) + org = org_m.group(1).strip() if org_m else "agnostic" + out.append((skill_md.parent.name, org, body)) + return out + + +# --------------------------------------------------------------------------- +# Scoring +# --------------------------------------------------------------------------- + + +def score_contracts(tools: list[ToolMeta]) -> list[ContractResult]: + """Compute the per-contract vendor-neutrality result.""" + seen = {c for t in tools for c in t.contracts} + unknown = seen - CONTRACT_POLICY.keys() + if unknown: + raise ValueError(f"contract(s) {sorted(unknown)} declared by a tool but absent from CONTRACT_POLICY") + + results: list[ContractResult] = [] + for contract, (klass, summary) in CONTRACT_POLICY.items(): + providers = [t for t in tools if contract in t.contracts] + interfaces = sorted(t.name for t in providers if t.kind == INTERFACE) + impls = [t for t in providers if t.kind == IMPLEMENTATION] + res = ContractResult( + contract=contract, + klass=klass, + summary=summary, + green=False, + basis="", + interfaces=interfaces, + implementations=sorted(impls, key=lambda t: t.vendor), + ) + if klass == AGNOSTIC: + res.green = True + res.basis = "vendor-neutral by construction — one spec serves every backend" + elif klass == SINGLE_ORG: + res.green = True + org = ", ".join(res.vendors) or "a single organisation" + res.basis = f"single-organisation capability ({org}); no vendor choice to make" + else: # vendor-backed + n = len(res.vendors) + res.green = n >= MIN_VENDORS + if res.green: + res.basis = f"{n} backend vendors: {', '.join(res.vendors)}" + elif n == 0: + res.basis = "no backend implemented yet" + else: + res.basis = ( + f"only {n} backend vendor ({', '.join(res.vendors)}); needs {MIN_VENDORS - n} more" + ) + results.append(res) + return results + + +def score_skills( + skills: list[tuple[str, str, str]], + contract_results: list[ContractResult], +) -> list[SkillResult]: + """Assess each skill: which capability contracts it invokes, and whether + any of those contracts has no alternative backend (a real lock-in).""" + green_by_contract = {r.contract: r.green for r in contract_results} + # A not-green vendor-backed contract with a single backend is a lock-in; + # name the sole vendor so the report can attribute it. + sole_vendor = { + r.contract: r.vendors[0] + for r in contract_results + if r.klass == VENDOR_BACKED and not r.green and len(r.vendors) == 1 + } + patterns = {c: [re.compile(p) for p in pats] for c, pats in CONTRACT_USAGE_TOKENS.items()} + out: list[SkillResult] = [] + for name, org, body in skills: + used = sorted(c for c, regexes in patterns.items() if any(r.search(body) for r in regexes)) + coupled = sorted( + (sole_vendor[c], c) for c in used if not green_by_contract.get(c, True) and c in sole_vendor + ) + if not used: + verdict = "capability-pure" + elif coupled: + verdict = "vendor-coupled" + else: + verdict = "portable" + out.append(SkillResult(name=name, organization=org, contracts=used, verdict=verdict, coupled=coupled)) + return out + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + +_MARK = {True: "✅", False: "❌"} + + +def _overall(contract_results: list[ContractResult]) -> tuple[int, int, int]: + green = sum(1 for r in contract_results if r.green) + total = len(contract_results) + pct = round(100 * green / total) if total else 0 + return green, total, pct + + +def render_text(contract_results: list[ContractResult], skill_results: list[SkillResult]) -> str: + green, total, pct = _overall(contract_results) + lines = [ + "Vendor-neutrality score (deterministic)", + "=" * 40, + f"Overall: {green}/{total} capability contracts vendor neutral ({pct}%)", + "", + "Per-contract status:", + ] + for r in contract_results: + lines.append(f" {_MARK[r.green]} {r.contract:26} [{r.klass:12}] {r.basis}") + # Skill summary + by_verdict: dict[str, int] = {} + by_org: dict[str, int] = {} + for s in skill_results: + by_verdict[s.verdict] = by_verdict.get(s.verdict, 0) + 1 + by_org[s.organization] = by_org.get(s.organization, 0) + 1 + lines += ["", f"Skills: {len(skill_results)} total"] + for verdict in ("capability-pure", "portable", "vendor-coupled"): + if verdict in by_verdict: + lines.append(f" {verdict:16} {by_verdict[verdict]}") + lines.append(" organization scope: " + ", ".join(f"{o}={n}" for o, n in sorted(by_org.items()))) + coupled = [s for s in skill_results if s.verdict == "vendor-coupled"] + if coupled: + lines += ["", "Vendor-coupled skills (no alternative backend for a named capability):"] + for s in coupled: + detail = "; ".join(f"{v} → {c}" for v, c in s.coupled) + lines.append(f" - {s.name}: {detail}") + return "\n".join(lines) + "\n" + + +def render_json(contract_results: list[ContractResult], skill_results: list[SkillResult]) -> str: + green, total, pct = _overall(contract_results) + payload = { + "overall": {"green": green, "total": total, "percent": pct}, + "contracts": [ + { + "contract": r.contract, + "class": r.klass, + "green": r.green, + "basis": r.basis, + "vendors": r.vendors, + "interfaces": r.interfaces, + "implementations": [{"tool": t.name, "vendor": t.vendor} for t in r.implementations], + } + for r in contract_results + ], + "skills": [ + { + "skill": s.name, + "organization": s.organization, + "contracts_used": s.contracts, + "verdict": s.verdict, + "coupled": [{"vendor": v, "contract": c} for v, c in s.coupled], + } + for s in skill_results + ], + } + return json.dumps(payload, indent=2, ensure_ascii=False) + + +def render_markdown(contract_results: list[ContractResult], skill_results: list[SkillResult]) -> str: + """Emit the generated block for docs/vendor-neutrality.md.""" + green, total, pct = _overall(contract_results) + lines = [ + f"**Overall vendor-neutrality score: {green}/{total} capability contracts " + f"({pct}%).** Generated by [`tools/vendor-neutrality-score`](../tools/vendor-neutrality-score/); " + "re-run it to refresh this section.", + "", + "| Capability contract | Neutral? | Class | Backends today | Basis |", + "|---|---|---|---|---|", + ] + for r in contract_results: + backends = ", ".join(r.vendors) if r.vendors else "—" + lines.append(f"| `{r.contract}` | {_MARK[r.green]} | {r.klass} | {backends} | {r.basis} |") + + by_verdict: dict[str, int] = {} + by_org: dict[str, int] = {} + for s in skill_results: + by_verdict[s.verdict] = by_verdict.get(s.verdict, 0) + 1 + by_org[s.organization] = by_org.get(s.organization, 0) + 1 + neutral = by_verdict.get("capability-pure", 0) + by_verdict.get("portable", 0) + lines += [ + "", + f"**Per-skill assessment: {neutral}/{len(skill_results)} skills carry no vendor lock-in.** " + "A skill is *capability-pure* when it names no backend at all, *portable* when every backend " + "it names has an alternative (its contract is green), and *vendor-coupled* only when it reaches " + "for a backend that is the sole implementation of a capability.", + "", + "| Skill neutrality | Count |", + "|---|---|", + f"| capability-pure (names no backend) | {by_verdict.get('capability-pure', 0)} |", + f"| portable (named backends are swappable) | {by_verdict.get('portable', 0)} |", + f"| vendor-coupled (sole-backend dependency) | {by_verdict.get('vendor-coupled', 0)} |", + "", + "Organization scope (declared, orthogonal to vendor): " + + ", ".join(f"{o} = {n}" for o, n in sorted(by_org.items())) + + ".", + ] + coupled = [s for s in skill_results if s.verdict == "vendor-coupled"] + if coupled: + lines += ["", "Vendor-coupled skills (the only lock-ins today):", ""] + for s in coupled: + detail = "; ".join(f"`{v}` → `{c}`" for v, c in s.coupled) + lines.append(f"- `{s.name}` — {detail}") + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def compute(repo_root: Path) -> tuple[list[ContractResult], list[SkillResult]]: + tools = load_tools(repo_root) + contract_results = score_contracts(tools) + skill_results = score_skills(load_skills(repo_root), contract_results) + return contract_results, skill_results + + +def main(argv: Iterable[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Deterministic Magpie vendor-neutrality score.") + parser.add_argument( + "--repo-root", type=Path, default=None, help="Repository root (default: auto-detect)." + ) + group = parser.add_mutually_exclusive_group() + group.add_argument("--json", action="store_true", help="Emit the full result as JSON.") + group.add_argument( + "--markdown", action="store_true", help="Emit the generated block for docs/vendor-neutrality.md." + ) + parser.add_argument( + "--fail-under", + type=int, + default=None, + metavar="PCT", + help="Exit non-zero if the overall percentage is below PCT (for CI gating).", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + repo_root = args.repo_root.resolve() if args.repo_root else find_repo_root() + try: + contract_results, skill_results = compute(repo_root) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + if args.json: + print(render_json(contract_results, skill_results)) + elif args.markdown: + print(render_markdown(contract_results, skill_results)) + else: + print(render_text(contract_results, skill_results), end="") + + if args.fail_under is not None: + _, _, pct = _overall(contract_results) + if pct < args.fail_under: + print(f"error: score {pct}% is below --fail-under {args.fail_under}%", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tools/vendor-neutrality-score/tests/__init__.py b/tools/vendor-neutrality-score/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py b/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py new file mode 100644 index 00000000..e61cb715 --- /dev/null +++ b/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py @@ -0,0 +1,200 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import json + +import pytest + +import vendor_neutrality_score as vns + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _tool(name: str, contract: str, kind: str, vendor: str) -> vns.ToolMeta: + return vns.ToolMeta(name=name, contracts=(contract,), kind=kind, vendor=vendor) + + +def _result(results: list[vns.ContractResult], contract: str) -> vns.ContractResult: + return next(r for r in results if r.contract == contract) + + +# --------------------------------------------------------------------------- +# Per-contract scoring +# --------------------------------------------------------------------------- + + +def test_vendor_backed_needs_two_distinct_vendors() -> None: + tools = [ + _tool("github", "contract:tracker", vns.IMPLEMENTATION, "GitHub"), + _tool("jira", "contract:tracker", vns.IMPLEMENTATION, "Atlassian"), + ] + r = _result(vns.score_contracts(tools), "contract:tracker") + assert r.green is True + assert r.vendors == ["Atlassian", "GitHub"] + + +def test_vendor_backed_single_vendor_is_not_green() -> None: + tools = [_tool("gmail", "contract:mail-draft", vns.IMPLEMENTATION, "Google")] + r = _result(vns.score_contracts(tools), "contract:mail-draft") + assert r.green is False + assert "needs 1 more" in r.basis + + +def test_same_vendor_twice_counts_once() -> None: + tools = [ + _tool("github", "contract:tracker", vns.IMPLEMENTATION, "GitHub"), + _tool("github-rollup", "contract:tracker", vns.IMPLEMENTATION, "GitHub"), + ] + r = _result(vns.score_contracts(tools), "contract:tracker") + assert r.green is False + assert r.vendors == ["GitHub"] + + +def test_interface_tools_do_not_count_as_a_backend() -> None: + tools = [ + _tool("cve-tool", "contract:cve-authority", vns.INTERFACE, "agnostic"), + _tool("cve-org", "contract:cve-authority", vns.IMPLEMENTATION, "CVE.org"), + ] + r = _result(vns.score_contracts(tools), "contract:cve-authority") + assert r.green is False # one interface + one impl = one backend vendor + assert r.interfaces == ["cve-tool"] + assert r.vendors == ["CVE.org"] + + +def test_agnostic_contract_is_green_with_only_an_interface() -> None: + tools = [_tool("scan-format", "contract:scan-format", vns.INTERFACE, "agnostic")] + r = _result(vns.score_contracts(tools), "contract:scan-format") + assert r.green is True + assert "construction" in r.basis + + +def test_single_org_contract_is_green_by_exemption() -> None: + tools = [_tool("apache-projects", "contract:project-metadata", vns.IMPLEMENTATION, "ASF")] + r = _result(vns.score_contracts(tools), "contract:project-metadata") + assert r.green is True + assert "ASF" in r.basis + + +def test_unknown_contract_raises() -> None: + tools = [_tool("mystery", "contract:does-not-exist", vns.IMPLEMENTATION, "X")] + with pytest.raises(ValueError, match="does-not-exist"): + vns.score_contracts(tools) + + +def test_all_policy_contracts_are_reported() -> None: + results = vns.score_contracts([]) + assert {r.contract for r in results} == set(vns.CONTRACT_POLICY) + + +# --------------------------------------------------------------------------- +# Per-skill assessment +# --------------------------------------------------------------------------- + + +def _contract_results_with_gap() -> list[vns.ContractResult]: + tools = [ + _tool("github", "contract:tracker", vns.IMPLEMENTATION, "GitHub"), + _tool("jira", "contract:tracker", vns.IMPLEMENTATION, "Atlassian"), + _tool("gmail", "contract:mail-draft", vns.IMPLEMENTATION, "Google"), + ] + return vns.score_contracts(tools) + + +def test_skill_capability_pure_when_no_backend_named() -> None: + skills = [("greeter", "agnostic", "This skill just talks to the user. No tools.")] + (s,) = vns.score_skills(skills, _contract_results_with_gap()) + assert s.verdict == "capability-pure" + assert s.contracts == [] + + +def test_skill_portable_when_named_contract_is_green() -> None: + skills = [("triage", "agnostic", "Run `gh pr list` then classify each PR.")] + (s,) = vns.score_skills(skills, _contract_results_with_gap()) + assert s.verdict == "portable" + assert s.contracts == ["contract:tracker"] + assert s.coupled == [] + + +def test_skill_vendor_coupled_on_sole_backend_contract() -> None: + skills = [("reply", "ASF", "Draft a courtesy reply with mcp__claude_ai_Gmail__create_draft.")] + (s,) = vns.score_skills(skills, _contract_results_with_gap()) + assert s.verdict == "vendor-coupled" + assert s.coupled == [("Google", "contract:mail-draft")] + + +def test_prose_mention_is_not_usage() -> None: + skills = [("docs", "agnostic", "Skills that read Gmail or GitHub archives are fine.")] + (s,) = vns.score_skills(skills, _contract_results_with_gap()) + assert s.verdict == "capability-pure" + + +# --------------------------------------------------------------------------- +# Rendering + CLI + integration +# --------------------------------------------------------------------------- + + +def test_json_render_is_valid_and_complete() -> None: + contracts = _contract_results_with_gap() + skills = vns.score_skills([("t", "agnostic", "gh pr view")], contracts) + payload = json.loads(vns.render_json(contracts, skills)) + assert payload["overall"]["total"] == len(vns.CONTRACT_POLICY) + assert any(c["contract"] == "contract:mail-draft" and c["green"] is False for c in payload["contracts"]) + + +def test_markdown_render_contains_score_and_table() -> None: + contracts = _contract_results_with_gap() + md = vns.render_markdown(contracts, []) + assert "Overall vendor-neutrality score" in md + assert "| `contract:tracker` |" in md + + +def test_compute_runs_against_the_live_repo() -> None: + root = vns.find_repo_root() + contract_results, skill_results = vns.compute(root) + assert len(contract_results) == len(vns.CONTRACT_POLICY) + assert len(skill_results) > 0 + # Every live contract carries a basis string. + assert all(r.basis for r in contract_results) + + +def test_doc_block_is_in_sync() -> None: + """The generated block in docs/vendor-neutrality.md must match the tool output.""" + root = vns.find_repo_root() + doc = (root / "docs" / "vendor-neutrality.md").read_text(encoding="utf-8") + begin = "" + assert begin in doc and end in doc, "regen markers missing from docs/vendor-neutrality.md" + block = doc.split(begin, 1)[1].split("-->", 1)[1].split(end, 1)[0].strip() + contract_results, skill_results = vns.compute(root) + expected = vns.render_markdown(contract_results, skill_results).strip() + assert block == expected, ( + "docs/vendor-neutrality.md is stale — regenerate with:\n" + " uv run --project tools/vendor-neutrality-score vendor-neutrality-score --markdown" + ) + + +def test_main_json_exits_zero() -> None: + assert vns.main(["--json"]) == 0 + + +def test_fail_under_gate() -> None: + # Far above any achievable score → non-zero exit. + assert vns.main(["--fail-under", "101"]) == 1 diff --git a/uv.lock b/uv.lock index 5bda18a1..25bf297d 100644 --- a/uv.lock +++ b/uv.lock @@ -35,6 +35,7 @@ members = [ "spec-status-index", "spec-validator", "symlink-lint", + "vendor-neutrality-score", "vulnogram-api", ] @@ -899,6 +900,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "vendor-neutrality-score" +version = "0.1.0" +source = { editable = "tools/vendor-neutrality-score" } + [[package]] name = "vulnogram-api" version = "0.1.0"