From d0fcffb214113e2887393fe5c152fcd9949e5b26 Mon Sep 17 00:00:00 2001 From: Matthew Watkins Date: Tue, 28 Jul 2026 11:53:42 +0100 Subject: [PATCH] Feat: Report zizmor findings at any severity The organisation scan pipeline is moving its zizmor floor from 'low' to 'informational' so every finding zizmor can report reaches the SARIF and code scanning. Match that here by cutting the zizmor category at INFORMATIONAL rather than LOW, so no published finding folds into the clean count. The previous cutoff assumed informational findings never reached the SARIF, which the 'low' floor guaranteed. That guarantee is going away: a scan of python-workflows produced 14 template-injection findings that the floor discarded, invisible to this report. zizmor emits both Low and Informational findings at SARIF level 'note', and the code-scanning alerts API exposes only that level, not the zizmor/severity property carried in the raw SARIF, so the two cannot be separated here. 'note' therefore still maps to LOW to avoid under-stating the estate, and the INFORMATIONAL cutoff makes the distinction moot for reporting: both tiers surface either way. The cutoff also closes a gap at SARIF level 'none', which normalises to INFORMATIONAL and previously passed silently. Add coverage pinning the zizmor cutoff and the unchanged MEDIUM global default, so a future change cannot silently reintroduce the gap. Co-authored-by: Claude Signed-off-by: Matthew Watkins --- src/github_security_report/categories.py | 22 ++++++++++++------ src/github_security_report/severity.py | 23 +++++++++++-------- tests/test_severity.py | 29 +++++++++++++++++++++--- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/github_security_report/categories.py b/src/github_security_report/categories.py index 3bc357b..58d0f1b 100644 --- a/src/github_security_report/categories.py +++ b/src/github_security_report/categories.py @@ -67,9 +67,9 @@ class CategoryMeta: # A repository fails (appears as an offender) only when it carries a finding # at or above this rung; findings below it fold into the clean count. The # global default is MEDIUM, so Low and Informational findings pass; a - # category may lower it (Zizmor uses LOW, so only Informational passes). - # Meaningful only for the severity-ranked signals; binary categories ignore - # it. Overridable per category via the JSON config. + # category may lower it (Zizmor uses INFORMATIONAL, so every finding + # counts). Meaningful only for the severity-ranked signals; binary + # categories ignore it. Overridable per category via the JSON config. fail_severity: Severity = Severity.MEDIUM @@ -106,10 +106,18 @@ class CategoryMeta: "Zizmor static analysis of GitHub Actions workflows, ranked " "worst-first by severity." ), - # zizmor emits its Low findings at SARIF level "note", which - # normalises to LOW (see severity.py), so any zizmor finding fails -- - # matching the ruleset-enforced PR gate that blocks on note-and-above. - fail_severity=Severity.LOW, + # The organisation scan pipeline runs zizmor with an + # 'informational' floor, so every finding it can report reaches the + # SARIF. Match that here: any zizmor finding counts, at any + # severity. This mirrors the ruleset-enforced PR gate, which blocks + # on any finding regardless of level. + # + # zizmor emits both Low and Informational findings at SARIF level + # "note", and the code-scanning alerts API exposes only that level + # (not zizmor's own severity property), so the two are + # indistinguishable here. Cutting at INFORMATIONAL sidesteps the + # ambiguity: both surface either way. + fail_severity=Severity.INFORMATIONAL, ), CategoryKey.AISLOP: CategoryMeta( key=CategoryKey.AISLOP, diff --git a/src/github_security_report/severity.py b/src/github_security_report/severity.py index da51d27..1bf218c 100644 --- a/src/github_security_report/severity.py +++ b/src/github_security_report/severity.py @@ -19,10 +19,13 @@ The ``note`` mapping mirrors zizmor's own SARIF encoder, which emits both its Low and Informational findings at SARIF level ``note`` (Medium -> ``warning``, High -> ``error``). The organisation scan pipeline runs zizmor with -``--min-severity low``, so informational findings never reach the uploaded -SARIF: every ``note`` alert in code scanning is a genuine Low finding, and the -ruleset-enforced PR gate blocks on it. Mapping ``note`` below LOW would -(and previously did) under-state the estate's posture relative to that gate. +``--min-severity informational``, so both tiers reach the uploaded SARIF and a +``note`` alert may be either. The code-scanning alerts API exposes only the +SARIF level -- not the ``zizmor/severity`` property carried in the raw SARIF -- +so the two cannot be separated here. ``note`` therefore stays at LOW rather +than INFORMATIONAL, which errs towards over- rather than under-stating the +estate's posture; the zizmor category cuts at INFORMATIONAL (see +``categories.py``) so both tiers are reported regardless of where this maps. """ from __future__ import annotations @@ -59,11 +62,13 @@ def label(self) -> str: } # SARIF level -> security scale, used only as a fallback (zizmor, aislop). -# zizmor's SARIF encoder emits Low AND Informational findings as ``note``, but -# the scan pipeline's --min-severity low floor keeps informational findings out -# of the SARIF entirely, so a ``note`` alert is a genuine Low finding (matching -# the ruleset-enforced PR gate, which blocks on note-and-above). aislop uses -# the same three levels. The rare ``none`` level stays at INFORMATIONAL. +# zizmor's SARIF encoder emits Low AND Informational findings as ``note``, and +# the scan pipeline's --min-severity informational floor lets both through, so +# a ``note`` alert may be either tier. The alerts API does not expose zizmor's +# own severity property, so they are indistinguishable here; ``note`` maps to +# LOW to avoid under-stating, and the zizmor category's INFORMATIONAL cutoff +# ensures both are reported. aislop uses the same three levels. The rare +# ``none`` level stays at INFORMATIONAL. _SARIF_LEVEL_NAMES: dict[str, Severity] = { "error": Severity.HIGH, "warning": Severity.MEDIUM, diff --git a/tests/test_severity.py b/tests/test_severity.py index 2323a9b..398253e 100644 --- a/tests/test_severity.py +++ b/tests/test_severity.py @@ -4,7 +4,8 @@ from __future__ import annotations -from github_security_report import severity +from github_security_report import categories, severity +from github_security_report.categories import CategoryKey from github_security_report.severity import Severity @@ -37,8 +38,10 @@ class TestSarifFallback: def test_sarif_levels(self) -> None: assert severity.from_sarif_level("error") is Severity.HIGH assert severity.from_sarif_level("warning") is Severity.MEDIUM - # zizmor emits its Low findings at SARIF level note; the scan - # pipeline's low floor keeps informational out of the SARIF. + # zizmor emits BOTH its Low and Informational findings at SARIF + # level note, and the alerts API does not expose which. note maps + # to LOW to avoid under-stating; the zizmor category's + # INFORMATIONAL cutoff ensures both tiers are still reported. assert severity.from_sarif_level("note") is Severity.LOW assert severity.from_sarif_level("none") is Severity.INFORMATIONAL @@ -47,6 +50,26 @@ def test_unknown(self) -> None: assert severity.from_sarif_level(None) is None +class TestCategoryFailSeverity: + """Severity floor at which each category counts a finding as a failure. + + Locked down because the zizmor cutoff is load-bearing: the organisation + scan pipeline runs zizmor with ``--min-severity informational``, and + zizmor emits Low and Informational alike at SARIF level ``note`` with + the code-scanning alerts API exposing no way to tell them apart. A + cutoff anywhere above INFORMATIONAL would therefore silently drop + genuine findings from the report. + """ + + def test_zizmor_counts_every_finding(self) -> None: + meta = categories.category_meta(CategoryKey.ZIZMOR) + assert meta.fail_severity is Severity.INFORMATIONAL + + def test_global_default_stays_medium(self) -> None: + meta = categories.category_meta(CategoryKey.CODEQL) + assert meta.fail_severity is Severity.MEDIUM + + class TestFromCodeScanning: def test_prefers_security_severity(self) -> None: # zizmor-style: only severity present