apache · potiuk · May 31, 2026 · May 31, 2026
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -409,6 +409,36 @@ repos:
         files: ^tools/permission-audit/(src|tests|pyproject\.toml)
         pass_filenames: false
 
+  # Project-local checks for the pre-flight classifier dry-run tool
+  # at `tools/preflight-audit/`. Lets us measure skip-rate before /
+  # after any rule edit in `bulk-mode.md`.
+  - repo: local
+    hooks:
+      - id: preflight-audit-ruff-check
+        name: ruff check (preflight-audit)
+        language: system
+        entry: uv run --directory tools/preflight-audit ruff check
+        files: ^tools/preflight-audit/(src|tests|pyproject\.toml)
+        pass_filenames: false
+      - id: preflight-audit-ruff-format
+        name: ruff format (preflight-audit)
+        language: system
+        entry: uv run --directory tools/preflight-audit ruff format --check
+        files: ^tools/preflight-audit/(src|tests|pyproject\.toml)
+        pass_filenames: false
+      - id: preflight-audit-mypy
+        name: mypy (preflight-audit)
+        language: system
+        entry: uv run --directory tools/preflight-audit mypy
+        files: ^tools/preflight-audit/(src|tests|pyproject\.toml)
+        pass_filenames: false
+      - id: preflight-audit-pytest
+        name: pytest (preflight-audit)
+        language: system
+        entry: uv run --directory tools/preflight-audit pytest
+        files: ^tools/preflight-audit/(src|tests|pyproject\.toml)
+        pass_filenames: false
+
   # Validate `.claude/skills/**`, every `tools/<name>/README.md`, and the
   # `docs/labels-and-capabilities.md` taxonomy via the
   # `skill-and-tool-validate` CLI. Re-fires on validator-source changes so

diff --git a/docs/labels-and-capabilities.md b/docs/labels-and-capabilities.md
@@ -187,6 +187,7 @@ Tools under [`tools/`](../tools/). Tools with two values (separated by
 | [`tools/ponymail`](../tools/ponymail/) | `capability:setup` + `capability:intake` | PonyMail archive substrate; same dual role as `mail-source` — substrate plus an intake-pipeline component |
 | [`tools/permission-audit`](../tools/permission-audit/) | `capability:setup` | Audit + atomically edit Claude Code `permissions.allow[]` entries; backs `/setup-steward verify --apply-permission-audit` (check 8d) |
 | [`tools/pr-management-stats`](../tools/pr-management-stats/) | `capability:stats` | PR-backlog analytics engine |
+| [`tools/preflight-audit`](../tools/preflight-audit/) | `capability:stats` | Dry-run the bulk-mode pre-flight classifier; measure skip-rate before / after any rule edit in the security-issue-sync skill |
 | [`tools/privacy-llm`](../tools/privacy-llm/) | `capability:setup` | Privacy-LLM PII-scrubbing gate |
 | [`tools/probe-templates`](../tools/probe-templates/) | `capability:setup` | Sandbox-doctor probe templates |
 | [`tools/sandbox-lint`](../tools/sandbox-lint/) | `capability:setup` | Sandbox settings linter |

diff --git a/tools/preflight-audit/README.md b/tools/preflight-audit/README.md
@@ -0,0 +1,136 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [preflight-audit](#preflight-audit)
+  - [Why](#why)
+  - [Invocation](#invocation)
+    - [Live mode](#live-mode)
+    - [Replay mode](#replay-mode)
+  - [Output](#output)
+  - [How the rules stay in sync](#how-the-rules-stay-in-sync)
+  - [Tuning workflow](#tuning-workflow)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+<!-- SPDX-License-Identifier: Apache-2.0
+     https://www.apache.org/licenses/LICENSE-2.0 -->
+
+# preflight-audit
+
+**Capability:** capability:stats
+
+Dry-run the bulk-mode pre-flight classifier against a real or
+replayed tracker. Use to **measure skip-rate before / after any
+rule change** — closes the tune-then-verify loop so rule edits
+are made against evidence, not guesswork.
+
+## Why
+
+The bulk-mode pre-flight classifier
+([`bulk-mode.md` § Pre-flight no-op classifier](../../.claude/skills/security-issue-sync/SKILL.md))
+decides whether to dispatch a subagent for each tracker in a
+bulk sync. Its rule table evolves as we learn how real adopter
+trackers behave. Each rule change needs a before / after
+measurement to know whether the change helped (more skips) or
+hurt (false-positive skips that miss real work).
+
+Without a tool, that measurement is a one-off Python script per
+PR — easy to lose, hard to reproduce, can't be wired into CI.
+This tool makes the measurement reusable.
+
+## Invocation
+
+```bash
+uv run --directory tools/preflight-audit preflight-audit classify [options]
+```
+
+### Live mode
+
+Fetch tracker state via `gh api graphql` and classify:
+
+```bash
+uv run --directory tools/preflight-audit preflight-audit classify \
+  --repo <owner>/<name> \
+  --issues 221,232,233,242,244
+```
+
+`--issues` is a comma-separated list of issue numbers (with or
+without `#` prefix). For the full `sync all` set, resolve the
+list yourself via `gh issue list --json number` and pipe the
+numbers in — the tool intentionally doesn't reimplement
+selector resolution (that's the sync skill's job).
+
+Pass `--bot-logins login1,login2` to extend the bot-equivalent
+login list for an adopter with personal-account bots.
+
+### Replay mode
+
+Classify a pre-fetched GraphQL response — for CI / regression
+testing without network calls:
+
+```bash
+uv run --directory tools/preflight-audit preflight-audit classify \
+  --load tests/fixtures/sample_response.json \
+  --now 2026-05-31T12:00:00Z
+```
+
+`--now` is required for replay mode to keep the classification
+deterministic (rules depend on "days ago" calculations).
+
+## Output
+
+Default is a human-readable grouped table:
+
+```text
+Total trackers: 43
+  dispatch: 29 (67%)
+  dispatch-urgent: 0 (0%)
+  skip-noop: 14 (32%)
+
+--- skip-noop (14) ---
+  # 221 OPEN   last=<bot> [skill]  → fix released; awaiting advisory; skill-last (1d)
+          labels: airflow+cve allocated+fix released
+  ...
+
+=== Estimated savings ===
+Subagents skipped: 14  → ~700 KB context saved (~175000 tokens @ 250 tok/KB)
+```
+
+Pass `--json` for machine-readable output (one object per
+tracker with number, decision, reason, label set).
+
+## How the rules stay in sync
+
+The classifier in
+[`src/preflight_audit/classifier.py`](src/preflight_audit/classifier.py)
+is the executable spec of the rule table in
+[`.claude/skills/security-issue-sync/bulk-mode.md`](../../.claude/skills/security-issue-sync/bulk-mode.md).
+The skill instructs the orchestrator how to apply the rules
+prose-by-prose; this tool implements them in code. Both forms
+describe the same rules and **must be edited in lock-step** —
+a PR that changes one should change the other.
+
+The tests in
+[`tests/test_classifier.py`](tests/test_classifier.py) cover
+each rule with a focused synthetic-input case. If the skill's
+rule table grows a new entry, add the matching `test_rule_N_*`
+case here.
+
+## Tuning workflow
+
+The intended workflow when changing a rule:
+
+1. Run `preflight-audit classify --repo <r> --issues <list>`
+   against your adopter tracker to capture the **before**
+   distribution.
+2. Edit the rule table in `bulk-mode.md` AND the matching
+   condition in `classifier.py`.
+3. Re-run the same command to capture the **after**
+   distribution.
+4. Cite both numbers in the PR body. (Strip adopter-specific
+   identifiers if the PR is public.)
+
+For tests, save the captured GraphQL response as a fixture and
+add a replay-mode test that asserts the expected classification
+breakdown — that's the eval-fixture pattern.
diff --git a/tools/preflight-audit/pyproject.toml b/tools/preflight-audit/pyproject.toml
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "preflight-audit"
+version = "0.1.0"
+description = "Dry-run the bulk-mode pre-flight classifier against a real or replayed tracker — measure skip-rate before / after any rule change."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "Apache-2.0" }
+# Stdlib-only at runtime; shells out to `gh` for live mode.
+dependencies = []
+
+[project.scripts]
+preflight-audit = "preflight_audit:main"
+
+[dependency-groups]
+dev = [
+  "mypy>=2.1.0",
+  "pytest>=8.0",
+  "ruff>=0.15.14",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/preflight_audit"]
+
+[tool.ruff]
+line-length = 110
+target-version = "py311"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = [
+  "E",
+  "W",
+  "F",
+  "I",
+  "B",
+  "UP",
+  "SIM",
+  "C4",
+  "RUF",
+]
+ignore = [
+  "E501",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["B", "SIM"]
+
+[tool.mypy]
+python_version = "3.11"
+files = ["src", "tests"]
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_unreachable = true
+check_untyped_defs = true
+no_implicit_optional = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+
+[tool.pytest.ini_options]
+minversion = "8.0"
+addopts = "-ra -q"
+testpaths = ["tests"]
diff --git a/tools/preflight-audit/src/preflight_audit/__init__.py b/tools/preflight-audit/src/preflight_audit/__init__.py
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from preflight_audit.classifier import (
+    Classification,
+    Decision,
+    IssueState,
+    classify_issue,
+    classify_response,
+)
+from preflight_audit.cli import main
+from preflight_audit.fetch import build_query, fetch_state
+
+__all__ = [
+    "Classification",
+    "Decision",
+    "IssueState",
+    "build_query",
+    "classify_issue",
+    "classify_response",
+    "fetch_state",
+    "main",
+]