Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions src/github_security_report/categories.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ class CategoryMeta:
# A repository fails (appears as an offender) only when it carries a finding
# at or above this rung; findings below it fold into the clean count. The
# global default is MEDIUM, so Low and Informational findings pass; a
# category may lower it (Zizmor uses LOW, so only Informational passes).
# Meaningful only for the severity-ranked signals; binary categories ignore
# it. Overridable per category via the JSON config.
# category may lower it (Zizmor uses INFORMATIONAL, so every finding
# counts). Meaningful only for the severity-ranked signals; binary
# categories ignore it. Overridable per category via the JSON config.
fail_severity: Severity = Severity.MEDIUM


Expand Down Expand Up @@ -106,10 +106,18 @@ class CategoryMeta:
"Zizmor static analysis of GitHub Actions workflows, ranked "
"worst-first by severity."
),
# zizmor emits its Low findings at SARIF level "note", which
# normalises to LOW (see severity.py), so any zizmor finding fails --
# matching the ruleset-enforced PR gate that blocks on note-and-above.
fail_severity=Severity.LOW,
# The organisation scan pipeline runs zizmor with an
# 'informational' floor, so every finding it can report reaches the
# SARIF. Match that here: any zizmor finding counts, at any
# severity. This mirrors the ruleset-enforced PR gate, which blocks
# on any finding regardless of level.
#
# zizmor emits both Low and Informational findings at SARIF level
# "note", and the code-scanning alerts API exposes only that level
# (not zizmor's own severity property), so the two are
# indistinguishable here. Cutting at INFORMATIONAL sidesteps the
# ambiguity: both surface either way.
fail_severity=Severity.INFORMATIONAL,
),
CategoryKey.AISLOP: CategoryMeta(
key=CategoryKey.AISLOP,
Expand Down
23 changes: 14 additions & 9 deletions src/github_security_report/severity.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@
The ``note`` mapping mirrors zizmor's own SARIF encoder, which emits both its
Low and Informational findings at SARIF level ``note`` (Medium -> ``warning``,
High -> ``error``). The organisation scan pipeline runs zizmor with
``--min-severity low``, so informational findings never reach the uploaded
SARIF: every ``note`` alert in code scanning is a genuine Low finding, and the
ruleset-enforced PR gate blocks on it. Mapping ``note`` below LOW would
(and previously did) under-state the estate's posture relative to that gate.
``--min-severity informational``, so both tiers reach the uploaded SARIF and a
``note`` alert may be either. The code-scanning alerts API exposes only the
SARIF level -- not the ``zizmor/severity`` property carried in the raw SARIF --
so the two cannot be separated here. ``note`` therefore stays at LOW rather
than INFORMATIONAL, which errs towards over- rather than under-stating the
estate's posture; the zizmor category cuts at INFORMATIONAL (see
``categories.py``) so both tiers are reported regardless of where this maps.
"""

from __future__ import annotations
Expand Down Expand Up @@ -59,11 +62,13 @@ def label(self) -> str:
}

# SARIF level -> security scale, used only as a fallback (zizmor, aislop).
# zizmor's SARIF encoder emits Low AND Informational findings as ``note``, but
# the scan pipeline's --min-severity low floor keeps informational findings out
# of the SARIF entirely, so a ``note`` alert is a genuine Low finding (matching
# the ruleset-enforced PR gate, which blocks on note-and-above). aislop uses
# the same three levels. The rare ``none`` level stays at INFORMATIONAL.
# zizmor's SARIF encoder emits Low AND Informational findings as ``note``, and
# the scan pipeline's --min-severity informational floor lets both through, so
# a ``note`` alert may be either tier. The alerts API does not expose zizmor's
# own severity property, so they are indistinguishable here; ``note`` maps to
# LOW to avoid under-stating, and the zizmor category's INFORMATIONAL cutoff
# ensures both are reported. aislop uses the same three levels. The rare
# ``none`` level stays at INFORMATIONAL.
_SARIF_LEVEL_NAMES: dict[str, Severity] = {
"error": Severity.HIGH,
"warning": Severity.MEDIUM,
Expand Down
29 changes: 26 additions & 3 deletions tests/test_severity.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@

from __future__ import annotations

from github_security_report import severity
from github_security_report import categories, severity
from github_security_report.categories import CategoryKey
from github_security_report.severity import Severity


Expand Down Expand Up @@ -37,8 +38,10 @@ class TestSarifFallback:
def test_sarif_levels(self) -> None:
assert severity.from_sarif_level("error") is Severity.HIGH
assert severity.from_sarif_level("warning") is Severity.MEDIUM
# zizmor emits its Low findings at SARIF level note; the scan
# pipeline's low floor keeps informational out of the SARIF.
# zizmor emits BOTH its Low and Informational findings at SARIF
# level note, and the alerts API does not expose which. note maps
# to LOW to avoid under-stating; the zizmor category's
# INFORMATIONAL cutoff ensures both tiers are still reported.
assert severity.from_sarif_level("note") is Severity.LOW
assert severity.from_sarif_level("none") is Severity.INFORMATIONAL

Expand All @@ -47,6 +50,26 @@ def test_unknown(self) -> None:
assert severity.from_sarif_level(None) is None


class TestCategoryFailSeverity:
"""Severity floor at which each category counts a finding as a failure.

Locked down because the zizmor cutoff is load-bearing: the organisation
scan pipeline runs zizmor with ``--min-severity informational``, and
zizmor emits Low and Informational alike at SARIF level ``note`` with
the code-scanning alerts API exposing no way to tell them apart. A
cutoff anywhere above INFORMATIONAL would therefore silently drop
genuine findings from the report.
"""

def test_zizmor_counts_every_finding(self) -> None:
meta = categories.category_meta(CategoryKey.ZIZMOR)
assert meta.fail_severity is Severity.INFORMATIONAL

def test_global_default_stays_medium(self) -> None:
meta = categories.category_meta(CategoryKey.CODEQL)
assert meta.fail_severity is Severity.MEDIUM


class TestFromCodeScanning:
def test_prefers_security_severity(self) -> None:
# zizmor-style: only severity present
Expand Down
Loading