diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 4afa3d9e727..2509b444724 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0951baf3154975d8a440cade065101df032fb2f800128f93719d2c0e5667d319","body_hash":"c3e3b328c5d10f596d0197ff6a67df16e271742259a40f2d7df0adad169a7336","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.63","copilot-sdk":"1.0.2"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ada1bd6ef00fbab34c49699cad06b920d53c24a70dbbc60a52a03b4f1ffd3745","body_hash":"5340aa7197e17d89569af296ccebab60805b8439bbb6cb95827eff1b24fa7d47","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.63","copilot-sdk":"1.0.2"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"4a3601121dd01d1626a1e23e37211e3254c1c06c","version":"v6.4.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"f9f3042f7e2789586610d6e8b85c8f03e5195baf","version":"v7.2.0"},{"repo":"docker/setup-buildx-action","sha":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5","version":"v4.1.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.7","digest":"sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7","digest":"sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.7","digest":"sha256:4757f198a3fa20f88bdbe70be7ae1a05f127d9c0a9e96a5d6460ef40c08fc83d","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.7@sha256:4757f198a3fa20f88bdbe70be7ae1a05f127d9c0a9e96a5d6460ef40c08fc83d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.7","digest":"sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.27","digest":"sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -589,7 +589,7 @@ jobs: - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download integrity-filtered logs - run: "mkdir -p /tmp/gh-aw/agent/integrity\n# Download logs filtered to only runs with DIFC integrity-filtered events.\n# --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking).\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --filtered-integrity --start-date -7d --json -c 200 \\\n --artifacts mcp --timeout 8 \\\n > /tmp/gh-aw/agent/integrity/filtered-logs.json || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' /tmp/gh-aw/agent/integrity/filtered-logs.json > /dev/null 2>&1; then\n echo \"⚠️ No valid logs produced; continuing with empty dataset\"\n echo '{\"runs\":[],\"summary\":{\"total_runs\":0}}' > /tmp/gh-aw/agent/integrity/filtered-logs.json\nfi\n\ncount=$(jq '.runs | length' /tmp/gh-aw/agent/integrity/filtered-logs.json 2>/dev/null || echo 0)\necho \"✅ Downloaded $count runs with integrity-filtered events\"\n" + run: "mkdir -p /tmp/gh-aw/agent/integrity\nmkdir -p /tmp/gh-aw/cache-memory/security-observability\n\nCACHE_FILE=/tmp/gh-aw/cache-memory/security-observability/filtered-logs.snapshot.json\nRUN_FILE=/tmp/gh-aw/agent/integrity/filtered-logs.json\nFRESH_LOGS=/tmp/gh-aw/agent/integrity/filtered-logs.fresh.json\nEMPTY_DATA='{\"runs\":[],\"summary\":{\"total_runs\":0}}'\nNOW_EPOCH=$(date +%s)\nMAX_CACHE_AGE_SECONDS=$((7 * 24 * 60 * 60))\n\n# Warm start from cached 7-day snapshot when available and fresh.\nif [ -f \"$CACHE_FILE\" ] && jq -e '.runs and .updated_at' \"$CACHE_FILE\" > /dev/null 2>&1; then\n cache_updated_at=$(jq -r '.updated_at' \"$CACHE_FILE\")\n cache_updated_epoch=$(\n date -d \"$cache_updated_at\" +%s 2>/dev/null \\\n || date -j -f \"%Y-%m-%dT%H:%M:%SZ\" \"$cache_updated_at\" +%s 2>/dev/null \\\n || echo 0\n )\n cache_age_seconds=$((NOW_EPOCH - cache_updated_epoch))\n if [ \"$cache_updated_epoch\" -gt 0 ] && [ \"$cache_age_seconds\" -le \"$MAX_CACHE_AGE_SECONDS\" ]; then\n jq '{runs: (.runs // []), summary: (.summary // {\"total_runs\": 0})}' \"$CACHE_FILE\" > \"$RUN_FILE\"\n echo \"✅ Warm cache restored (${cache_age_seconds}s old)\"\n else\n echo \"⚠️ Cache snapshot is stale (${cache_age_seconds}s old); starting fresh\"\n fi\nfi\n\n# Download logs filtered to only runs with DIFC integrity-filtered events.\n# --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking).\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --filtered-integrity --start-date -7d --json -c 200 \\\n --artifacts mcp --timeout 8 \\\n > \"$FRESH_LOGS\" || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' \"$FRESH_LOGS\" > /dev/null 2>&1; then\n echo \"⚠️ No valid logs produced; continuing with empty dataset\"\n echo \"$EMPTY_DATA\" > \"$FRESH_LOGS\"\nfi\n\n# Merge warm-start and fresh runs; fresh entries override warm-cache entries with the same run_id.\nif [ -f \"$RUN_FILE\" ] && jq -e '.runs' \"$RUN_FILE\" > /dev/null 2>&1; then\n jq -s '\n {\n runs: (\n ((.[0].runs // []) + (.[1].runs // []))\n | sort_by(.run_id)\n | group_by(.run_id)\n | map(.[-1])\n ),\n summary: {\n total_runs: (\n ((.[0].runs // []) + (.[1].runs // []) | sort_by(.run_id) | group_by(.run_id) | length)\n )\n }\n }\n ' \"$RUN_FILE\" \"$FRESH_LOGS\" > \"$RUN_FILE.merged\"\n mv \"$RUN_FILE.merged\" \"$RUN_FILE\"\nelse\n mv \"$FRESH_LOGS\" \"$RUN_FILE\"\nfi\n\ncount=$(jq '.runs | length' \"$RUN_FILE\" 2>/dev/null || echo 0)\necho \"✅ Downloaded $count runs with integrity-filtered events\"\n\n# Persist updated 7-day snapshot back to cache-memory every run.\njq -n \\\n --arg updated_at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \\\n --slurpfile payload \"$RUN_FILE\" \\\n '{\n updated_at: $updated_at,\n runs: ($payload[0].runs // []),\n summary: {\n total_runs: (($payload[0].runs // []) | length)\n }\n }' > \"$CACHE_FILE\"\n" # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory diff --git a/.github/workflows/daily-security-observability.md b/.github/workflows/daily-security-observability.md index b8ac7289390..8aacd86d7eb 100644 --- a/.github/workflows/daily-security-observability.md +++ b/.github/workflows/daily-security-observability.md @@ -37,22 +37,82 @@ steps: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p /tmp/gh-aw/agent/integrity + mkdir -p /tmp/gh-aw/cache-memory/security-observability + + CACHE_FILE=/tmp/gh-aw/cache-memory/security-observability/filtered-logs.snapshot.json + RUN_FILE=/tmp/gh-aw/agent/integrity/filtered-logs.json + FRESH_LOGS=/tmp/gh-aw/agent/integrity/filtered-logs.fresh.json + EMPTY_DATA='{"runs":[],"summary":{"total_runs":0}}' + NOW_EPOCH=$(date +%s) + MAX_CACHE_AGE_SECONDS=$((7 * 24 * 60 * 60)) + + # Warm start from cached 7-day snapshot when available and fresh. + if [ -f "$CACHE_FILE" ] && jq -e '.runs and .updated_at' "$CACHE_FILE" > /dev/null 2>&1; then + cache_updated_at=$(jq -r '.updated_at' "$CACHE_FILE") + cache_updated_epoch=$( + date -d "$cache_updated_at" +%s 2>/dev/null \ + || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$cache_updated_at" +%s 2>/dev/null \ + || echo 0 + ) + cache_age_seconds=$((NOW_EPOCH - cache_updated_epoch)) + if [ "$cache_updated_epoch" -gt 0 ] && [ "$cache_age_seconds" -le "$MAX_CACHE_AGE_SECONDS" ]; then + jq '{runs: (.runs // []), summary: (.summary // {"total_runs": 0})}' "$CACHE_FILE" > "$RUN_FILE" + echo "✅ Warm cache restored (${cache_age_seconds}s old)" + else + echo "⚠️ Cache snapshot is stale (${cache_age_seconds}s old); starting fresh" + fi + fi + # Download logs filtered to only runs with DIFC integrity-filtered events. # --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking). # --timeout 8: cap execution at 8 minutes to prevent runaway downloads. gh aw logs --filtered-integrity --start-date -7d --json -c 200 \ --artifacts mcp --timeout 8 \ - > /tmp/gh-aw/agent/integrity/filtered-logs.json || true + > "$FRESH_LOGS" || true # Validate JSON output and fall back to an empty dataset on failure - if ! jq -e '.runs' /tmp/gh-aw/agent/integrity/filtered-logs.json > /dev/null 2>&1; then + if ! jq -e '.runs' "$FRESH_LOGS" > /dev/null 2>&1; then echo "⚠️ No valid logs produced; continuing with empty dataset" - echo '{"runs":[],"summary":{"total_runs":0}}' > /tmp/gh-aw/agent/integrity/filtered-logs.json + echo "$EMPTY_DATA" > "$FRESH_LOGS" fi - count=$(jq '.runs | length' /tmp/gh-aw/agent/integrity/filtered-logs.json 2>/dev/null || echo 0) + # Merge warm-start and fresh runs; fresh entries override warm-cache entries with the same run_id. + if [ -f "$RUN_FILE" ] && jq -e '.runs' "$RUN_FILE" > /dev/null 2>&1; then + jq -s ' + { + runs: ( + ((.[0].runs // []) + (.[1].runs // [])) + | sort_by(.run_id) + | group_by(.run_id) + | map(.[-1]) + ), + summary: { + total_runs: ( + ((.[0].runs // []) + (.[1].runs // []) | sort_by(.run_id) | group_by(.run_id) | length) + ) + } + } + ' "$RUN_FILE" "$FRESH_LOGS" > "$RUN_FILE.merged" + mv "$RUN_FILE.merged" "$RUN_FILE" + else + mv "$FRESH_LOGS" "$RUN_FILE" + fi + + count=$(jq '.runs | length' "$RUN_FILE" 2>/dev/null || echo 0) echo "✅ Downloaded $count runs with integrity-filtered events" + # Persist updated 7-day snapshot back to cache-memory every run. + jq -n \ + --arg updated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --slurpfile payload "$RUN_FILE" \ + '{ + updated_at: $updated_at, + runs: ($payload[0].runs // []), + summary: { + total_runs: (($payload[0].runs // []) | length) + } + }' > "$CACHE_FILE" + tools: bash: - "*" @@ -174,9 +234,14 @@ Upload both charts using `upload_asset` and record the returned URLs. ## Phase 3: Collect DIFC Integrity-Filtered Events -### Step 3.1: Check for DIFC Data +### Step 3.1: Warm Start Validation + DIFC Data Check + +The startup step restores a cached snapshot from `/tmp/gh-aw/cache-memory/security-observability/filtered-logs.snapshot.json` before collecting fresh runs. -Read `/tmp/gh-aw/agent/integrity/filtered-logs.json`. If the `runs` array is empty or missing (no runs found in the last 7 days), note "No DIFC integrity-filtered events found in the last 7 days." and proceed directly to Phase 5 (combined report). +1. Verify the restored snapshot age using `updated_at` from the cache file: + - If age is `<= 7 days`, treat it as a valid warm start. + - If age is `> 7 days` or missing, treat it as stale and rely on fresh logs. +2. Read `/tmp/gh-aw/agent/integrity/filtered-logs.json`. If the `runs` array is empty or missing (no runs found in the last 7 days), note "No DIFC integrity-filtered events found in the last 7 days." and proceed directly to Phase 5 (combined report). ### Step 3.2: Fetch Detailed DIFC Gateway Data