diff --git a/docs/src/content/docs/specs/forecast-specification.md b/docs/src/content/docs/specs/forecast-specification.md index 3bc75cb2810..cffb03ecd30 100644 --- a/docs/src/content/docs/specs/forecast-specification.md +++ b/docs/src/content/docs/specs/forecast-specification.md @@ -931,6 +931,8 @@ and adding new fixtures. - **T-FC-021**: Sampling respects `--days` historical window cutoff. - **T-FC-022**: Run with missing `aw_info.json` artifact contributes zero ET and is still counted in `sampled_runs`. - **T-FC-023**: Workflow with zero sampled runs produces nil projection with zero fields. +- **T-FC-024**: An in-progress run with a non-zero token usage snapshot is represented as a partial observation. +- **T-ET-006**: A run with total effective tokens of at least 1,000,000 is handled without overflow. #### 12.1.4 Monte Carlo Engine Tests @@ -972,6 +974,7 @@ and adding new fixtures. | Data sampling with limit and window | T-FC-020–021 | 1 | Required | | Missing artifact graceful handling | T-FC-022 | 1 | Required | | Nil projection for empty sample | T-FC-023 | 1 | Required | +| Partial observation for in-progress run | T-FC-024 | 1 | Required | | Knuth Poisson algorithm (λ ≤ 15) | T-FC-031 | 1 | Required | | Normal approximation (λ > 15) | T-FC-032 | 1 | Required | | Zero-λ projection | T-FC-033 | 1 | Required | diff --git a/pkg/cli/forecast_compliance_fixtures_formal_test.go b/pkg/cli/forecast_compliance_fixtures_formal_test.go index a52247ab4ae..caa7198c2de 100644 --- a/pkg/cli/forecast_compliance_fixtures_formal_test.go +++ b/pkg/cli/forecast_compliance_fixtures_formal_test.go @@ -22,7 +22,7 @@ package cli // - FC-P6: run_summary_failed.json has conclusion == "failure" (T-FC-035) // - FC-P7: run_summary_cancelled.json has conclusion == "cancelled" (T-FC-036) // - FC-P8: RunSummary JSON round-trip serialization is lossless -// - FC-P9: run_started_at <= updated_at in every fixture +// - FC-P9: StartedAt <= UpdatedAt in every fixture // - FC-P10: all five fixtures have required Monte Carlo input fields import ( @@ -69,10 +69,10 @@ func TestFormal_P1_FixtureFieldMapping(t *testing.T) { // Top-level identity field. assert.Contains(t, fixture, "run_id", "P1: run_id must be present") - // run sub-object must expose conclusion, updated_at, run_started_at. + // run sub-object must expose conclusion, updatedAt, startedAt. run, ok := fixture["run"].(map[string]any) require.True(t, ok, "P1: 'run' must be a JSON object") - for _, field := range []string{"conclusion", "updated_at", "run_started_at"} { + for _, field := range []string{"conclusion", "updatedAt", "startedAt"} { assert.Contains(t, run, field, "P1: run.%s must be present for forecast inputs", field) } @@ -411,7 +411,7 @@ func TestFormal_P13_FixtureJSONConformance(t *testing.T) { // run sub-object required fields. run, ok := fixture["run"].(map[string]any) require.True(t, ok, "P13: 'run' must be a JSON object") - runRequired := []string{"conclusion", "updated_at", "run_started_at"} + runRequired := []string{"conclusion", "updatedAt", "startedAt"} for _, field := range runRequired { assert.Contains(t, run, field, "P13: run.%q must be present for duration and Bernoulli derivation", field) @@ -504,6 +504,26 @@ func TestFormal_FC_P7_CancelledRunFixture(t *testing.T) { "so it is included in the sample but not counted as a Bernoulli success") } +// TestFormal_FC_P11_PartialETFixture verifies that run_summary_partial_et.json +// represents an in-progress run with a non-zero token usage snapshot. +// +// Specification reference: T-FC-024; specs/forecast-compliance-fixtures/README.md +func TestFormal_FC_P11_PartialETFixture(t *testing.T) { + fixture := loadFixture(t, "run_summary_partial_et.json") + + run, ok := fixture["run"].(map[string]any) + require.True(t, ok, "FC-P11: 'run' must be a JSON object") + assert.Equal(t, "in_progress", run["status"], + "FC-P11 (T-FC-024): partial fixture must represent an in-progress run") + + usage, ok := fixture["token_usage_summary"].(map[string]any) + require.True(t, ok, "FC-P11: token_usage_summary must be a JSON object") + et, ok := usage["total_effective_tokens"].(float64) + require.True(t, ok, "FC-P11: total_effective_tokens must be a number") + assert.Greater(t, et, 0.0, + "FC-P11 (T-FC-024): partial fixture must contain a non-zero token usage snapshot") +} + // TestFormal_FC_P8_RunSummaryRoundTrip verifies that marshalling a RunSummary to // JSON and unmarshalling it back produces an equal value (cache-hit determinism). // @@ -549,10 +569,10 @@ func TestFormal_FC_P8_RunSummaryRoundTrip(t *testing.T) { "FC-P8: Run.UpdatedAt must survive round-trip") } -// TestFormal_FC_P9_TimestampOrdering verifies that run_started_at <= updated_at +// TestFormal_FC_P9_TimestampOrdering verifies that StartedAt <= UpdatedAt // in every fixture (TLA+ ordering invariant). // -// Formal predicate (FC-P9): ∀f ∈ Fixtures: f["run"]["run_started_at"] ≤ f["run"]["updated_at"] +// Formal predicate (FC-P9): ∀f ∈ Fixtures: f.Run.StartedAt ≤ f.Run.UpdatedAt // Specification reference: §6.2.2 Duration Derivation func TestFormal_FC_P9_TimestampOrdering(t *testing.T) { fixtures := []string{ @@ -561,28 +581,25 @@ func TestFormal_FC_P9_TimestampOrdering(t *testing.T) { "run_summary_failed.json", "run_summary_high_et.json", "run_summary_cancelled.json", + "run_summary_partial_et.json", } for _, name := range fixtures { t.Run(name, func(t *testing.T) { - fixture := loadFixture(t, name) - - run, ok := fixture["run"].(map[string]any) - require.True(t, ok, "FC-P9: 'run' must be a JSON object in %s", name) - - startedStr, ok := run["run_started_at"].(string) - require.True(t, ok, "FC-P9: run.run_started_at must be a string in %s", name) - updatedStr, ok := run["updated_at"].(string) - require.True(t, ok, "FC-P9: run.updated_at must be a string in %s", name) - - started, err := time.Parse(time.RFC3339, startedStr) - require.NoError(t, err, "FC-P9: run_started_at must parse as RFC3339 in %s", name) - updated, err := time.Parse(time.RFC3339, updatedStr) - require.NoError(t, err, "FC-P9: updated_at must parse as RFC3339 in %s", name) - - assert.False(t, started.After(updated), - "FC-P9: run_started_at (%s) must be <= updated_at (%s) in %s", - startedStr, updatedStr, name) + data, err := os.ReadFile(filepath.Join(fixtureDir(t), name)) + require.NoError(t, err, "FC-P9: fixture file %q must be readable", name) + + var summary RunSummary + require.NoError(t, json.Unmarshal(data, &summary), + "FC-P9: fixture file %q must unmarshal as RunSummary", name) + require.False(t, summary.Run.StartedAt.IsZero(), + "FC-P9: run.startedAt must populate RunSummary.Run.StartedAt in %s", name) + require.False(t, summary.Run.UpdatedAt.IsZero(), + "FC-P9: run.updatedAt must populate RunSummary.Run.UpdatedAt in %s", name) + + assert.False(t, summary.Run.StartedAt.After(summary.Run.UpdatedAt), + "FC-P9: StartedAt (%s) must be <= UpdatedAt (%s) in %s", + summary.Run.StartedAt, summary.Run.UpdatedAt, name) }) } } @@ -607,6 +624,7 @@ func TestFormal_FC_P10_MonteCarloInputCompleteness(t *testing.T) { {name: "run_summary_failed.json", wantConclusion: "failure", aicMustBeGT0: false}, {name: "run_summary_high_et.json", wantConclusion: "success", aicMustBeGT0: true}, {name: "run_summary_cancelled.json", wantConclusion: "cancelled", aicMustBeGT0: false}, + {name: "run_summary_partial_et.json", wantConclusion: "", aicMustBeGT0: true}, } for _, tc := range cases { @@ -663,6 +681,7 @@ var documentedForecastFixtures = []string{ "run_summary_failed.json", "run_summary_high_et.json", "run_summary_cancelled.json", + "run_summary_partial_et.json", } // TestFormal_FixtureCountConsistency verifies that the fixture files documented in diff --git a/pkg/workflow/github_mcp_access_control_formal_test.go b/pkg/workflow/github_mcp_access_control_formal_test.go index dc085535281..d26c2b6f6cb 100644 --- a/pkg/workflow/github_mcp_access_control_formal_test.go +++ b/pkg/workflow/github_mcp_access_control_formal_test.go @@ -505,6 +505,7 @@ var documentedComplianceFixtures = []string{ "empty-repos-block.yaml", "role-deny.yaml", "tool-name-filter.yaml", + "empty-tool-name-deny.yaml", "blocked-user-deny.yaml", "private-repo-block.yaml", "integrity-level-block.yaml", diff --git a/specs/awf-config-sources-compliance/README.md b/specs/awf-config-sources-compliance/README.md index ccb7fb1189f..1787727c788 100644 --- a/specs/awf-config-sources-compliance/README.md +++ b/specs/awf-config-sources-compliance/README.md @@ -30,6 +30,7 @@ The following test IDs cover the `DriftRecord` schema and its usage requirements ## Spec Reference - **Specification**: `specs/awf-config-sources-spec.md` +- **Repository structure**: [Structure](../awf-config-sources-spec.md#structure) - **Defining section**: §6.5 — DriftRecord Entity Schema - **Related sections**: §6.2 (Drift Detection Procedure), §5 (Conformance Requirements CR-05, CR-06) @@ -40,7 +41,7 @@ The following test IDs cover the `DriftRecord` schema and its usage requirements Conformance tests that validate `DriftRecord` schema compliance are implemented in: ``` -pkg/workflow/awf_config_drift_test.go — DriftRecord schema validation and usage (T-DR-001 through T-DR-010) +pkg/workflow/awf_config_drift_test.go — DriftRecord schema validation and usage (T-DR-001 through T-DR-010; T-DR-005: TestDriftRecord_TDR005_NoAdditionalProperties) ``` To run related tests: diff --git a/specs/awf-config-sources-spec.md b/specs/awf-config-sources-spec.md index f9ec7256b2d..469946fc9d0 100644 --- a/specs/awf-config-sources-spec.md +++ b/specs/awf-config-sources-spec.md @@ -40,9 +40,20 @@ The following documents are authoritative and MUST be consulted together: - `docs/authentication-architecture.md` — credential isolation architecture - `schemas/README.md` — schema directory overview +## Structure + +This specification, the [conformance fixture index](awf-config-sources-compliance/README.md), and +[`pkg/workflow/awf_config_drift_test.go`](../pkg/workflow/awf_config_drift_test.go) form one +conformance unit. The specification defines `DriftRecord` requirements, the fixture index maps +them to `T-DR-*` IDs, and the Go test file implements those IDs. Changes to any member of this +unit **MUST** keep the other two members synchronized. + ## 3. Data Model -This section defines the canonical data entities used in the drift detection procedure. The `DriftRecord` entity is the primary structured output of drift detection. +This section defines the canonical data entities used in the drift detection procedure. The +repository relationships that maintain its conformance coverage are defined in +[Structure](#structure). The `DriftRecord` entity is the primary structured output of drift +detection. ### 3.1 DriftRecord diff --git a/specs/compiler-threat-detection-spec.md b/specs/compiler-threat-detection-spec.md index 74c742f1724..88452ac134c 100644 --- a/specs/compiler-threat-detection-spec.md +++ b/specs/compiler-threat-detection-spec.md @@ -79,18 +79,11 @@ This section anchors the specification version to the minimum gh-aw binary versi | Spec version | Minimum gh-aw binary version | Lock-file compatibility notes | |--------------|------------------------------|-------------------------------| | `1.0.20` | `v0.83.6` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), conditional import rejection (`imports.if` rejection for CTR-020), `workflow_run` trigger branch scope enforcement (CTR-021), git subprocess argument-injection guards for remote import/download ref and path arguments (CTR-022), and bash command allowlist illusion rejection for engines lacking allowlist enforcement (CTR-023). No `.lock.yml` schema changes are introduced by CTR-022 or CTR-023; both are compile-time-only validations. | -| `1.0.19` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), conditional import rejection (`imports.if` rejection for CTR-020), and `workflow_run` trigger branch scope enforcement (CTR-021). The `docker-sbx` runtime enforcement (CTR-004 scope) requires `sudo: true`, compatible runner topology, and a minimum AWF version; the credential refresh step emitted before agent execution is a security improvement with no new constraint on `.lock.yml` semantics. Playwright CLI mode (`tools.playwright.mode: cli`) is compiler-generated infrastructure with no new constraint on `.lock.yml` semantics. | -| `1.0.18` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), conditional import rejection (`imports.if` rejection for CTR-020), and `workflow_run` trigger branch scope enforcement (CTR-021). The `docker-sbx` runtime enforcement (CTR-004 scope) requires `sudo: true`, compatible runner topology, and a minimum AWF version; the credential refresh step emitted before agent execution is a security improvement with no new constraint on `.lock.yml` semantics. Playwright CLI mode (`tools.playwright.mode: cli`) is compiler-generated infrastructure with no new constraint on `.lock.yml` semantics. | -| `1.0.17` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), conditional import rejection (`imports.if` rejection for CTR-020), and `workflow_run` trigger branch scope enforcement (CTR-021). The `docker-sbx` runtime enforcement (CTR-004 scope) requires `sudo: true`, compatible runner topology, and a minimum AWF version; the credential refresh step emitted before agent execution is a security improvement with no new constraint on `.lock.yml` semantics. | -| `1.0.16` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), conditional import rejection (`imports.if` rejection for CTR-020), and `workflow_run` trigger branch scope enforcement (CTR-021). The `docker-sbx` runtime enforcement (CTR-004 scope) requires `sudo: true`, compatible runner topology, and a minimum AWF version; the credential refresh step emitted before agent execution is a security improvement with no new constraint on `.lock.yml` semantics. | -| `1.0.15` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), conditional import rejection (`imports.if` rejection for CTR-020), and `workflow_run` trigger branch scope enforcement (CTR-021). | -| `1.0.14` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019), and conditional import rejection (`imports.if` rejection for CTR-020). | -| `1.0.13` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), and cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019). | -| `1.0.12` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), and cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019). | -| `1.0.11` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), and cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019). | -| `1.0.10` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016), update-check validation (`check-for-updates` handling for CTR-018), and cache-memory integrity enforcement (`update_cache_memory` gating for CTR-019). | -| `1.0.9` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016) and update-check validation (`check-for-updates` handling for CTR-018). Top-level `sandbox: false` is no longer a valid workflow input; `sandbox.agent: false` is the supported field for CTR-004 detection. | -| `1.0.8` | `v0.72.1` (or newer) | Threat-detection behavior must remain compatible with current `.lock.yml` compilation semantics, including manifest drift enforcement (`gh-aw-manifest` checks for CTR-016) and update-check validation (`check-for-updates` handling for CTR-018). | +| `1.0.15`–`1.0.19` | `v0.72.1` (or newer) | Adds `workflow_run` trigger branch-scope enforcement (CTR-021); runtime-only `docker-sbx`, credential-refresh, and Playwright changes introduce no `.lock.yml` schema constraint. | +| `1.0.8`–`1.0.14` baseline | `v0.72.1` (or newer) | Establishes manifest drift (CTR-016), update-check (CTR-018), cache-memory integrity (CTR-019), and conditional-import (CTR-020) validation. | +Compact changelog: `1.0.8` introduced CTR-016 and CTR-018; `1.0.10`–`1.0.13` added +CTR-019; `1.0.14` added CTR-020; `1.0.15` added CTR-021; and `1.0.20` added CTR-022 +and CTR-023. Versions with no distinct lock-file impact are grouped above. When this specification version changes, maintainers MUST update this table in the same pull request as any lock-file compatibility changes. @@ -247,6 +240,7 @@ When the GitHub API or any external service required by the optimizer (for examp 1. The optimizer **MUST** not emit false noop reports. When authoritative data cannot be retrieved, the optimizer **MUST** emit an `OPTIMIZER_DEGRADED` diagnostic entry in its daily output that records the failing endpoint(s), the HTTP status or error class, and the UTC timestamp of the failure. 2. The optimizer **MUST NOT** open a pull request or update spec artifacts based on incomplete threat-coverage data obtained during a degraded API run. 3. The optimizer **SHOULD** retry failed API calls with an exponential back-off policy (initial delay: 10 seconds; maximum delay: 5 minutes; maximum attempts: 3) before declaring the run degraded. +4. After exhausting that policy, the optimizer **MUST NOT** retry the same request against a different degraded endpoint; it **MUST** declare the run degraded unless a configured, independently authoritative endpoint is available. **Failure Mode 2 — Runner Timeout** diff --git a/specs/forecast-compliance-fixtures/README.md b/specs/forecast-compliance-fixtures/README.md index 5b9a83785a4..17a45e6832a 100644 --- a/specs/forecast-compliance-fixtures/README.md +++ b/specs/forecast-compliance-fixtures/README.md @@ -12,7 +12,7 @@ This fixture represents a single successful workflow run (`daily-report`) with: - `conclusion: "success"` — the run is counted as successful in Bernoulli sampling - `token_usage_summary.total_effective_tokens: 5400` — the ET observation used in bootstrap resampling -- `run.updated_at` and `run.run_started_at` — used to compute `duration_seconds` +- `run.updatedAt` and `run.startedAt` — used to compute `duration_seconds` Use this fixture as the baseline for Monte Carlo engine compliance tests (**T-FC-031** through **T-FC-040**) by loading it as a cached run summary. @@ -48,8 +48,8 @@ The `run_summary_minimal.json` fixture follows the `RunSummary` struct defined i | JSON Field | Go Field | Forecast Usage | |---|---|---| | `run.conclusion` | `Run.Conclusion` | Bernoulli success probability | -| `run.updated_at` | `Run.UpdatedAt` | Duration computation | -| `run.run_started_at` | `Run.RunStartedAt` | Duration computation | +| `run.updatedAt` | `Run.UpdatedAt` | Duration computation | +| `run.startedAt` | `Run.StartedAt` | Duration computation | | `token_usage_summary.total_effective_tokens` | `TokenUsage.TotalEffectiveTokens` | Bootstrap ET sample | | `run_id` | `RunID` | Run identification | @@ -65,7 +65,8 @@ To add a fixture covering a specific compliance scenario: | Fixture Name | Purpose | Test IDs | |---|---|---| -| `run_summary_zero_et.json` | Run with missing/zero ET (artifact not downloaded) | T-FC-022 | -| `run_summary_failed.json` | Run with `conclusion: "failure"` for Bernoulli sampling | T-FC-035 | -| `run_summary_high_et.json` | Run with very high ET (≥ 1,000,000) for overflow checks | T-ET-006 | -| `run_summary_cancelled.json` | Run with `conclusion: "cancelled"` (included in sample but not a Bernoulli success; ET is zero because the run did not complete) | T-FC-036 | +| `run_summary_zero_et.json` | Run with missing/zero ET (artifact not downloaded) | [T-FC-022](../../docs/src/content/docs/specs/forecast-specification.md#1213-data-sampling-tests) | +| `run_summary_failed.json` | Run with `conclusion: "failure"` for Bernoulli sampling | [T-FC-035](../../docs/src/content/docs/specs/forecast-specification.md#1214-monte-carlo-engine-tests) | +| `run_summary_high_et.json` | Run with very high ET (≥ 1,000,000) for overflow checks | [T-ET-006](../../docs/src/content/docs/specs/forecast-specification.md#1213-data-sampling-tests) | +| `run_summary_cancelled.json` | Run with `conclusion: "cancelled"` (included in sample but not a Bernoulli success; ET is zero because the run did not complete) | [T-FC-036](../../docs/src/content/docs/specs/forecast-specification.md#1214-monte-carlo-engine-tests) | +| `run_summary_partial_et.json` | In-progress run with a non-zero token usage snapshot | [T-FC-024](../../docs/src/content/docs/specs/forecast-specification.md#1213-data-sampling-tests) | diff --git a/specs/forecast-compliance-fixtures/run_summary_cancelled.json b/specs/forecast-compliance-fixtures/run_summary_cancelled.json index 27d48102c7c..1a0c0a0a878 100644 --- a/specs/forecast-compliance-fixtures/run_summary_cancelled.json +++ b/specs/forecast-compliance-fixtures/run_summary_cancelled.json @@ -10,9 +10,9 @@ "head_sha": "abc123def456abc123def456abc123def456abc123", "status": "completed", "conclusion": "cancelled", - "created_at": "2026-05-01T11:00:00Z", - "updated_at": "2026-05-01T11:02:10Z", - "run_started_at": "2026-05-01T11:00:05Z" + "createdAt": "2026-05-01T11:00:00Z", + "updatedAt": "2026-05-01T11:02:10Z", + "startedAt": "2026-05-01T11:00:05Z" }, "metrics": { "total_steps": 2, diff --git a/specs/forecast-compliance-fixtures/run_summary_failed.json b/specs/forecast-compliance-fixtures/run_summary_failed.json index 93b4462c2c6..6b364f042a2 100644 --- a/specs/forecast-compliance-fixtures/run_summary_failed.json +++ b/specs/forecast-compliance-fixtures/run_summary_failed.json @@ -10,9 +10,9 @@ "head_sha": "abc123def456abc123def456abc123def456abc123", "status": "completed", "conclusion": "failure", - "created_at": "2026-05-01T11:00:00Z", - "updated_at": "2026-05-01T11:05:30Z", - "run_started_at": "2026-05-01T11:00:05Z" + "createdAt": "2026-05-01T11:00:00Z", + "updatedAt": "2026-05-01T11:05:30Z", + "startedAt": "2026-05-01T11:00:05Z" }, "metrics": { "total_steps": 5, diff --git a/specs/forecast-compliance-fixtures/run_summary_high_et.json b/specs/forecast-compliance-fixtures/run_summary_high_et.json index d4f99847c78..23845be8d46 100644 --- a/specs/forecast-compliance-fixtures/run_summary_high_et.json +++ b/specs/forecast-compliance-fixtures/run_summary_high_et.json @@ -10,9 +10,9 @@ "head_sha": "abc123def456abc123def456abc123def456abc123", "status": "completed", "conclusion": "success", - "created_at": "2026-05-01T11:00:00Z", - "updated_at": "2026-05-01T11:05:30Z", - "run_started_at": "2026-05-01T11:00:05Z" + "createdAt": "2026-05-01T11:00:00Z", + "updatedAt": "2026-05-01T11:05:30Z", + "startedAt": "2026-05-01T11:00:05Z" }, "metrics": { "total_steps": 5, diff --git a/specs/forecast-compliance-fixtures/run_summary_minimal.json b/specs/forecast-compliance-fixtures/run_summary_minimal.json index d541fbc36e8..ee8ca7ad6b8 100644 --- a/specs/forecast-compliance-fixtures/run_summary_minimal.json +++ b/specs/forecast-compliance-fixtures/run_summary_minimal.json @@ -10,9 +10,9 @@ "head_sha": "abc123def456abc123def456abc123def456abc123", "status": "completed", "conclusion": "success", - "created_at": "2026-05-01T11:00:00Z", - "updated_at": "2026-05-01T11:05:30Z", - "run_started_at": "2026-05-01T11:00:05Z" + "createdAt": "2026-05-01T11:00:00Z", + "updatedAt": "2026-05-01T11:05:30Z", + "startedAt": "2026-05-01T11:00:05Z" }, "metrics": { "total_steps": 5, diff --git a/specs/forecast-compliance-fixtures/run_summary_partial_et.json b/specs/forecast-compliance-fixtures/run_summary_partial_et.json new file mode 100644 index 00000000000..0897655a23a --- /dev/null +++ b/specs/forecast-compliance-fixtures/run_summary_partial_et.json @@ -0,0 +1,56 @@ +{ + "cli_version": "0.0.0-test", + "run_id": 12345679, + "processed_at": "2026-05-01T12:00:00Z", + "run": { + "id": 12345679, + "name": "daily-report", + "workflow_id": 98765, + "head_branch": "main", + "head_sha": "abc123def456abc123def456abc123def456abc124", + "status": "in_progress", + "conclusion": "", + "createdAt": "2026-05-01T11:00:00Z", + "updatedAt": "2026-05-01T11:02:30Z", + "startedAt": "2026-05-01T11:00:05Z" + }, + "metrics": { + "total_steps": 2, + "failed_steps": 0, + "tool_calls": 6 + }, + "access_analysis": null, + "firewall_analysis": null, + "redacted_domains_analysis": null, + "missing_tools": [], + "missing_data": [], + "noops": [], + "mcp_failures": [], + "artifacts_list": [], + "job_details": [], + "token_usage_summary": { + "total_input_tokens": 2300, + "total_output_tokens": 450, + "total_cache_read_tokens": 300, + "total_cache_write_tokens": 75, + "total_requests": 2, + "total_duration_ms": 8500, + "total_response_bytes": 12000, + "cache_efficiency": 0.1154, + "total_effective_tokens": 2750, + "total_aic": 0.00275, + "by_model": { + "claude-3-7-sonnet": { + "provider": "anthropic", + "input_tokens": 2300, + "output_tokens": 450, + "cache_read_tokens": 300, + "cache_write_tokens": 75, + "requests": 2, + "duration_ms": 8500, + "response_bytes": 12000, + "effective_tokens": 2750 + } + } + } +} diff --git a/specs/forecast-compliance-fixtures/run_summary_zero_et.json b/specs/forecast-compliance-fixtures/run_summary_zero_et.json index a23782ae395..82044cc4f5a 100644 --- a/specs/forecast-compliance-fixtures/run_summary_zero_et.json +++ b/specs/forecast-compliance-fixtures/run_summary_zero_et.json @@ -10,9 +10,9 @@ "head_sha": "abc123def456abc123def456abc123def456abc123", "status": "completed", "conclusion": "success", - "created_at": "2026-05-01T11:00:00Z", - "updated_at": "2026-05-01T11:05:30Z", - "run_started_at": "2026-05-01T11:00:05Z" + "createdAt": "2026-05-01T11:00:00Z", + "updatedAt": "2026-05-01T11:05:30Z", + "startedAt": "2026-05-01T11:00:05Z" }, "metrics": { "total_steps": 5, diff --git a/specs/github-mcp-access-control-compliance/README.md b/specs/github-mcp-access-control-compliance/README.md index e6b889ae7b0..9267b51139f 100644 --- a/specs/github-mcp-access-control-compliance/README.md +++ b/specs/github-mcp-access-control-compliance/README.md @@ -3,6 +3,8 @@ This directory contains fixture stubs for the Section 11 compliance tests of the [GitHub MCP Access Control Specification](../../scratchpad/github-mcp-access-control-specification.md). +**Spec version pinned at commit `2c1cfd71010a2d1ab9d9149118beb076d2098d7d`.** + Each fixture describes a test scenario with an input tool configuration and the expected access-control decision. Fixtures are consumed by the compliance test runner to verify that implementations satisfy the normative requirements in §§4–10 of the specification. @@ -73,6 +75,7 @@ The denial code is selected by the first failing guard in the evaluation order a | `empty-repos-block.yaml` | Empty `repos` array is rejected at compile time | T-GH-015, T-GH-016 | | `role-deny.yaml` | Role filter denies access when user role is insufficient | T-GH-019, T-GH-020 | | `tool-name-filter.yaml` | `allowed-tools` filter allows or denies by tool name | T-GH-031, T-GH-032, T-GH-033 | +| `empty-tool-name-deny.yaml` | Empty tool name is denied against a non-empty `allowed-tools` list | P1_ToolAllowed | | `blocked-user-deny.yaml` | `blocked-users` denies listed actors unconditionally | T-GH-071, T-GH-072 | | `private-repo-block.yaml` | `private-repos: false` blocks access to private repository | T-GH-024, T-GH-025 | | `integrity-level-block.yaml` | `min-integrity: approved` blocks content below the threshold | T-GH-051, T-GH-052 | @@ -148,3 +151,6 @@ Formal conformance tests are implemented in: The test suite includes: - **Predicate-mapped tests** (`TestFormal_*`) — each test maps to a specific guard predicate (P1–P6) or invariant documented in the Formal Model section above. - **Fixture runner** (`TestFormal_FixtureRunner`) — loads every YAML fixture file from this directory and drives each scenario through the formal evaluator. This ensures the fixture files, error codes, and expected decisions remain consistent with the formal model. + +The `combined-blocked-integrity.yaml` fixture verifies that the runner returns P5's `-32005` +before P6's `-32006` when both guards fail. diff --git a/specs/github-mcp-access-control-compliance/empty-tool-name-deny.yaml b/specs/github-mcp-access-control-compliance/empty-tool-name-deny.yaml new file mode 100644 index 00000000000..3ab0e3b6581 --- /dev/null +++ b/specs/github-mcp-access-control-compliance/empty-tool-name-deny.yaml @@ -0,0 +1,26 @@ +# Empty Tool Name Denial — Compliance Fixture +# Spec: §4.5.3 Tool selection + +fixture_id: "empty-tool-name-deny" +description: > + A request with an empty tool name MUST be denied when `allowed-tools` is non-empty. + +spec_refs: + - "§4.5.3 — P1_ToolAllowed is evaluated first" + +scenarios: + - scenario_id: "empty-tool-name-deny" + description: "Empty tool name is not present in a non-empty allowed-tools list" + input: + tool_config: + repos: + - "*/*" + allowed-tools: + - "issue_read" + request: + repository: "example/repo" + tool_name: "" + expected: + decision: deny + error_code: -32001 + reason: "tool not in allowed-tools list"