From d22a69dda678ce2da8d6fa71584d8ded75585ecc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:23:56 +0000 Subject: [PATCH 1/2] Initial plan From 78f3a1b77531f4ab85a3206512e67f4e952ac530 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:38:35 +0000 Subject: [PATCH 2/2] reorder cache-memory and repo-memory restore steps before user steps Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../workflows/agentic-token-audit.lock.yml | 20 +++---- .../agentic-token-optimizer.lock.yml | 18 +++---- .../workflows/api-consumption-report.lock.yml | 26 +++++----- .github/workflows/audit-workflows.lock.yml | 26 +++++----- .../aw-failure-investigator.lock.yml | 12 ++--- .github/workflows/ci-coach.lock.yml | 32 ++++++------ .github/workflows/ci-doctor.lock.yml | 32 ++++++------ .../workflows/copilot-agent-analysis.lock.yml | 18 +++---- .../copilot-centralization-optimizer.lock.yml | 20 +++---- .github/workflows/copilot-opt.lock.yml | 32 ++++++------ .../copilot-pr-merged-report.lock.yml | 18 +++---- .../copilot-pr-nlp-analysis.lock.yml | 52 +++++++++---------- .../copilot-pr-prompt-analysis.lock.yml | 18 +++---- .../copilot-session-insights.lock.yml | 46 ++++++++-------- .github/workflows/daily-code-metrics.lock.yml | 30 +++++------ .../daily-community-attribution.lock.yml | 20 +++---- .../daily-experiment-report.lock.yml | 32 ++++++------ .github/workflows/daily-fact.lock.yml | 10 ++-- .github/workflows/daily-hippo-learn.lock.yml | 12 ++--- .../workflows/daily-issues-report.lock.yml | 32 ++++++------ .github/workflows/daily-news.lock.yml | 52 +++++++++---------- .../daily-performance-summary.lock.yml | 26 +++++----- .../workflows/daily-repo-chronicle.lock.yml | 30 +++++------ .../daily-safe-output-optimizer.lock.yml | 10 ++-- .../daily-security-observability.lock.yml | 32 ++++++------ .../workflows/daily-sentrux-report.lock.yml | 16 +++--- .../daily-spdd-spec-planner.lock.yml | 10 ++-- .../dataflow-pr-discussion-dataset.lock.yml | 52 +++++++++---------- .github/workflows/dead-code-remover.lock.yml | 6 +-- .github/workflows/deep-report.lock.yml | 52 +++++++++---------- .github/workflows/dependabot-burner.lock.yml | 12 ++--- .../detection-analysis-report.lock.yml | 26 +++++----- .../github-mcp-structural-analysis.lock.yml | 30 +++++------ .../workflows/glossary-maintainer.lock.yml | 6 +-- .github/workflows/go-logger.lock.yml | 24 ++++----- .github/workflows/gpclean.lock.yml | 12 ++--- .github/workflows/hippo-embed.lock.yml | 16 +++--- .github/workflows/jsweep.lock.yml | 8 +-- .github/workflows/org-health-report.lock.yml | 30 +++++------ .github/workflows/portfolio-analyst.lock.yml | 30 +++++------ .../prompt-clustering-analysis.lock.yml | 46 ++++++++-------- .github/workflows/python-data-charts.lock.yml | 30 +++++------ .../repository-quality-improver.lock.yml | 6 +-- .github/workflows/safe-output-health.lock.yml | 10 ++-- .../workflows/slide-deck-maintainer.lock.yml | 22 ++++---- .../workflows/stale-repo-identifier.lock.yml | 32 ++++++------ .../workflows/static-analysis-report.lock.yml | 24 ++++----- .../workflows/step-name-alignment.lock.yml | 6 +-- .github/workflows/super-linter.lock.yml | 12 ++--- .../workflows/technical-doc-writer.lock.yml | 18 +++---- .github/workflows/unbloat-docs.lock.yml | 32 ++++++------ .../weekly-blog-post-writer.lock.yml | 20 +++---- .../workflows/weekly-issue-summary.lock.yml | 30 +++++------ .../workflow-health-manager.lock.yml | 10 ++-- pkg/workflow/compiler_yaml_main_job.go | 16 +++--- pkg/workflow/compiler_yaml_main_job_test.go | 52 +++++++++++++++++++ 56 files changed, 704 insertions(+), 648 deletions(-) diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index eb6a5914d3a..39748b2668c 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -512,6 +512,16 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone repo-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: memory/token-audit + TARGET_REPO: ${{ github.repository }} + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Setup Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -527,16 +537,6 @@ jobs: name: Download agentic workflow logs run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\n# Download last 24 hours of agentic workflow logs as JSON\n# Allow partial results — gh aw logs streams incrementally, so even if\n# it hits an API rate limit partway through, the JSON written so far is\n# still valid and should be processed by the agent.\nLOGS_EXIT=0\ngh aw logs \\\n --start-date -1d \\\n --json \\\n -c 100 \\\n > /tmp/gh-aw/token-audit/workflow-logs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/workflow-logs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/workflow-logs.json)\n echo \"✅ Downloaded $TOTAL agentic workflow runs (last 24 hours)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"⚠️ gh aw logs exited with code $LOGS_EXIT (partial results — likely API rate limit)\"\n fi\nelse\n echo \"❌ No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/workflow-logs.json\nfi" - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone repo-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: memory/token-audit - TARGET_REPO: ${{ github.repository }} - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index b8eb3df4b52..8db94931cca 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -476,15 +476,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download recent agentic workflow logs - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\necho \"📥 Downloading agentic workflow logs (last 7 days)...\"\n\nLOGS_EXIT=0\ngh aw logs \\\n --start-date -7d \\\n --json \\\n -c 50 \\\n > /tmp/gh-aw/token-audit/all-runs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/all-runs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/all-runs.json)\n echo \"✅ Downloaded $TOTAL agentic workflow runs (last 7 days)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"⚠️ gh aw logs exited with code $LOGS_EXIT (partial results — likely API rate limit)\"\n fi\nelse\n echo \"❌ No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/all-runs.json\nfi\n" - - name: Aggregate top workflows by AIC usage - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\njq '{\n generated_at: (now | todateiso8601),\n window_days: 7,\n top_workflows: (\n [.runs[]\n | select(.status == \"completed\")\n | {\n workflow_name: .workflow_name,\n aic: (.aic // 0),\n raw_tokens: (.token_usage // 0),\n turns: (.turns // 0),\n action_minutes: (.action_minutes // 0)\n }\n ]\n | group_by(.workflow_name)\n | map({\n workflow_name: .[0].workflow_name,\n run_count: length,\n total_aic: (map(.aic) | add),\n avg_aic: ((map(.aic) | add) / length),\n total_raw_tokens: (map(.raw_tokens) | add),\n total_turns: (map(.turns) | add),\n total_action_minutes: (map(.action_minutes) | add)\n })\n | sort_by(.total_aic)\n | reverse\n | .[:10]\n )\n}' /tmp/gh-aw/token-audit/all-runs.json > /tmp/gh-aw/token-audit/top-workflows.json\n\necho \"✅ Generated top workflow summary at /tmp/gh-aw/token-audit/top-workflows.json\"\njq '.top_workflows' /tmp/gh-aw/token-audit/top-workflows.json\n" - - name: Load optimization history - run: "set -euo pipefail\n\nOPT_LOG=\"/tmp/gh-aw/repo-memory/default/optimization-log.json\"\nif [ -f \"$OPT_LOG\" ]; then\n echo \"✅ Previous optimizations:\"\n jq -r '.[] | \"\\(.date): \\(.workflow_name)\"' \"$OPT_LOG\"\nelse\n echo \"ℹ️ No previous optimization history found.\"\nfi" - # Repo memory git-based storage configuration from frontmatter processed below - name: Clone repo-memory branch (default) env: @@ -495,6 +486,15 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download recent agentic workflow logs + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\necho \"📥 Downloading agentic workflow logs (last 7 days)...\"\n\nLOGS_EXIT=0\ngh aw logs \\\n --start-date -7d \\\n --json \\\n -c 50 \\\n > /tmp/gh-aw/token-audit/all-runs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/all-runs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/all-runs.json)\n echo \"✅ Downloaded $TOTAL agentic workflow runs (last 7 days)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"⚠️ gh aw logs exited with code $LOGS_EXIT (partial results — likely API rate limit)\"\n fi\nelse\n echo \"❌ No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/all-runs.json\nfi\n" + - name: Aggregate top workflows by AIC usage + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\njq '{\n generated_at: (now | todateiso8601),\n window_days: 7,\n top_workflows: (\n [.runs[]\n | select(.status == \"completed\")\n | {\n workflow_name: .workflow_name,\n aic: (.aic // 0),\n raw_tokens: (.token_usage // 0),\n turns: (.turns // 0),\n action_minutes: (.action_minutes // 0)\n }\n ]\n | group_by(.workflow_name)\n | map({\n workflow_name: .[0].workflow_name,\n run_count: length,\n total_aic: (map(.aic) | add),\n avg_aic: ((map(.aic) | add) / length),\n total_raw_tokens: (map(.raw_tokens) | add),\n total_turns: (map(.turns) | add),\n total_action_minutes: (map(.action_minutes) | add)\n })\n | sort_by(.total_aic)\n | reverse\n | .[:10]\n )\n}' /tmp/gh-aw/token-audit/all-runs.json > /tmp/gh-aw/token-audit/top-workflows.json\n\necho \"✅ Generated top workflow summary at /tmp/gh-aw/token-audit/top-workflows.json\"\njq '.top_workflows' /tmp/gh-aw/token-audit/top-workflows.json\n" + - name: Load optimization history + run: "set -euo pipefail\n\nOPT_LOG=\"/tmp/gh-aw/repo-memory/default/optimization-log.json\"\nif [ -f \"$OPT_LOG\" ]; then\n echo \"✅ Previous optimizations:\"\n jq -r '.[] | \"\\(.date): \\(.workflow_name)\"' \"$OPT_LOG\"\nelse\n echo \"ℹ️ No previous optimization history found.\"\nfi" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/api-consumption-report.lock.yml b/.github/workflows/api-consumption-report.lock.yml index 88708a2a684..72b3bee93fc 100644 --- a/.github/workflows/api-consumption-report.lock.yml +++ b/.github/workflows/api-consumption-report.lock.yml @@ -773,19 +773,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: trending-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -802,6 +789,19 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: trending-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index b5a6a9b1636..9ae5124a232 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -584,19 +584,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: trending-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -623,6 +610,19 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Setup Python environment + run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: trending-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/aw-failure-investigator.lock.yml b/.github/workflows/aw-failure-investigator.lock.yml index f46396282b8..15385587510 100644 --- a/.github/workflows/aw-failure-investigator.lock.yml +++ b/.github/workflows/aw-failure-investigator.lock.yml @@ -552,18 +552,18 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Deterministic pre-fetch for failure analysis - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent/failure-investigator\npython3 - <<'PY'\nimport json\nimport os\nimport subprocess\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\nfrom urllib.parse import urlencode\n\nREPO = os.environ[\"GITHUB_REPOSITORY\"]\nOUT = \"/tmp/gh-aw/agent/failure-investigator/prefetch.json\"\nTRACKER_ID = \"aw-failure-investigator\"\nLOOKBACK_HOURS = 6\nFAILURE_CONCLUSIONS = {\"failure\", \"timed_out\", \"startup_failure\", \"cancelled\"}\nMAX_DISCOVERY_PAGES = 20\n# Most dominant signatures appear in the final 30-60 lines.\nMAX_LOG_TAIL_LINES = 50\n# Deep-dive budget: investigate at most this many distinct failed runs.\nMAX_FAILURES_TO_DETAIL = 5\nAGENTIC_WORKFLOW_PATHS = {\n f\".github/workflows/{path.name}\"\n for path in Path(\".github/workflows\").glob(\"*.lock.yml\")\n}\n\ndef cmd_display(args):\n return \" \".join(args)\n\ndef run_json(args):\n try:\n out = subprocess.check_output(args, text=True, stderr=subprocess.STDOUT)\n return json.loads(out)\n except subprocess.CalledProcessError as error:\n print(f\"Warning: command failed: {cmd_display(args)}\")\n print(error.output)\n return None\n except json.JSONDecodeError as error:\n print(f\"Warning: non-JSON output from command: {cmd_display(args)} ({error})\")\n return None\n except OSError as error:\n print(f\"Warning: could not execute command: {cmd_display(args)} ({error})\")\n return None\n\ndef run_text(args):\n try:\n return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as error:\n print(f\"Warning: command failed: {cmd_display(args)}\")\n print(error.output)\n return \"\"\n except OSError as error:\n print(f\"Warning: could not execute command: {cmd_display(args)} ({error})\")\n return \"\"\n\ndef run_api_json(endpoint, params):\n query = urlencode(params)\n return run_json([\"gh\", \"api\", f\"{endpoint}?{query}\"])\n\ndef is_failure_conclusion(conclusion):\n return (conclusion or \"\").lower() in FAILURE_CONCLUSIONS\n\ndef normalize_workflow_path(path):\n return (path or \"\").split(\"@\", 1)[0]\n\ndef is_agentic_workflow_path(path):\n workflow_path = normalize_workflow_path(path)\n if AGENTIC_WORKFLOW_PATHS:\n return workflow_path in AGENTIC_WORKFLOW_PATHS\n print(\"Warning: no local .lock.yml workflows found; falling back to workflow path suffix matching\")\n return workflow_path.endswith(\".lock.yml\")\n\ndef isoformat_z(dt):\n return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace(\"+00:00\", \"Z\")\n\ndef list_failed_agentic_runs():\n created_since = isoformat_z(datetime.now(timezone.utc) - timedelta(hours=LOOKBACK_HOURS))\n page = 1\n failed_runs = []\n\n while True:\n response = run_api_json(\n f\"repos/{REPO}/actions/runs\",\n {\n \"exclude_pull_requests\": \"true\",\n \"status\": \"completed\",\n \"created\": f\">={created_since}\",\n \"per_page\": \"100\",\n \"page\": str(page),\n },\n ) or {}\n workflow_runs = response.get(\"workflow_runs\") or []\n if not workflow_runs:\n break\n\n for run in workflow_runs:\n workflow_path = normalize_workflow_path(run.get(\"path\"))\n if not is_agentic_workflow_path(workflow_path):\n continue\n if not is_failure_conclusion(run.get(\"conclusion\")):\n continue\n\n failed_runs.append(\n {\n \"run_id\": run.get(\"id\"),\n \"workflow_name\": run.get(\"name\"),\n \"workflow_path\": workflow_path,\n \"created_at\": run.get(\"created_at\"),\n \"status\": run.get(\"status\"),\n \"conclusion\": run.get(\"conclusion\"),\n \"url\": run.get(\"html_url\"),\n }\n )\n\n if len(workflow_runs) < 100:\n break\n if page >= MAX_DISCOVERY_PAGES:\n print(f\"Warning: reached pagination cap ({MAX_DISCOVERY_PAGES} pages) while listing workflow runs\")\n break\n page += 1\n\n failed_runs.sort(key=lambda run: run.get(\"created_at\") or \"\", reverse=True)\n return failed_runs\n\nfailed_runs = list_failed_agentic_runs()\n\n# Cap the number of runs to detail so the payload stays compact.\nfailure_details = []\nfor run in failed_runs[:MAX_FAILURES_TO_DETAIL]:\n run_id = run.get(\"run_id\")\n if not run_id:\n continue\n\n run_view = run_json(\n [\n \"gh\",\n \"run\",\n \"view\",\n str(run_id),\n \"--repo\",\n REPO,\n \"--json\",\n \"databaseId,url,name,workflowName,createdAt,conclusion,status,jobs\",\n ]\n )\n if not run_view:\n continue\n\n failed_job_names = []\n failed_steps = []\n truncated_error_logs = []\n agent_job_conclusion = None\n for job in run_view.get(\"jobs\", []):\n job_name = job.get(\"name\")\n job_conclusion = (job.get(\"conclusion\") or \"\").lower()\n if (job_name or \"\").lower() == \"agent\":\n agent_job_conclusion = job_conclusion or None\n\n if is_failure_conclusion(job_conclusion):\n if job_name:\n failed_job_names.append(job_name)\n for step in job.get(\"steps\", []):\n if is_failure_conclusion(step.get(\"conclusion\")):\n failed_steps.append(\n {\n \"job_id\": job.get(\"databaseId\"),\n \"job_name\": job_name,\n \"step_name\": step.get(\"name\"),\n }\n )\n\n job_id = job.get(\"databaseId\")\n if job_id:\n log_text = run_text(\n [\n \"gh\",\n \"run\",\n \"view\",\n str(run_id),\n \"--repo\",\n REPO,\n \"--job\",\n str(job_id),\n \"--log-failed\",\n ]\n )\n if log_text:\n tail_lines = log_text.splitlines()[-MAX_LOG_TAIL_LINES:]\n truncated_error_logs.append(\n {\n \"job_id\": job_id,\n \"job_name\": job_name,\n \"line_count\": len(tail_lines),\n \"tail_lines\": \"\\n\".join(tail_lines),\n }\n )\n\n failure_details.append(\n {\n \"run_id\": run_id,\n \"workflow_name\": run_view.get(\"workflowName\") or run_view.get(\"name\"),\n \"workflow_path\": run.get(\"workflow_path\"),\n \"url\": run_view.get(\"url\"),\n \"created_at\": run_view.get(\"createdAt\"),\n \"status\": run_view.get(\"status\"),\n \"conclusion\": run_view.get(\"conclusion\"),\n \"failed_job_names\": sorted(set(failed_job_names)),\n \"agent_job_conclusion\": agent_job_conclusion,\n \"failed_steps\": failed_steps,\n \"truncated_error_logs\": truncated_error_logs,\n }\n )\n\nexisting_tracking_issues = run_json(\n [\n \"gh\",\n \"issue\",\n \"list\",\n \"--repo\",\n REPO,\n \"--state\",\n \"open\",\n \"--search\",\n f\"gh-aw-tracker-id: {TRACKER_ID}\",\n \"--limit\",\n \"100\",\n \"--json\",\n \"number,title,state,url,labels,createdAt,updatedAt\",\n ]\n) or []\n\npayload = {\n \"generated_at\": datetime.now(timezone.utc).isoformat(),\n \"repository\": REPO,\n \"lookback_window\": f\"{LOOKBACK_HOURS}h\",\n \"failed_run_ids\": [run.get(\"run_id\") for run in failed_runs if run.get(\"run_id\")],\n \"failures\": failure_details,\n \"existing_tracking_issues\": existing_tracking_issues,\n}\n\nwith open(OUT, \"w\", encoding=\"utf-8\") as f:\n json.dump(payload, f, indent=2)\n f.write(\"\\n\")\n\nprint(f\"Wrote deterministic prefetch payload to {OUT}\")\nprint(f\"Failed runs in payload: {len(payload['failed_run_ids'])}\")\nprint(f\"Existing tracking issues in payload: {len(existing_tracking_issues)}\")\nPY" - # Cache configuration from frontmatter processed below - name: Failure investigator prefetch uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: aw-failure-investigator-prefetch-${{ github.run_id }} path: /tmp/gh-aw/agent/failure-investigator + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Deterministic pre-fetch for failure analysis + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent/failure-investigator\npython3 - <<'PY'\nimport json\nimport os\nimport subprocess\nfrom datetime import datetime, timedelta, timezone\nfrom pathlib import Path\nfrom urllib.parse import urlencode\n\nREPO = os.environ[\"GITHUB_REPOSITORY\"]\nOUT = \"/tmp/gh-aw/agent/failure-investigator/prefetch.json\"\nTRACKER_ID = \"aw-failure-investigator\"\nLOOKBACK_HOURS = 6\nFAILURE_CONCLUSIONS = {\"failure\", \"timed_out\", \"startup_failure\", \"cancelled\"}\nMAX_DISCOVERY_PAGES = 20\n# Most dominant signatures appear in the final 30-60 lines.\nMAX_LOG_TAIL_LINES = 50\n# Deep-dive budget: investigate at most this many distinct failed runs.\nMAX_FAILURES_TO_DETAIL = 5\nAGENTIC_WORKFLOW_PATHS = {\n f\".github/workflows/{path.name}\"\n for path in Path(\".github/workflows\").glob(\"*.lock.yml\")\n}\n\ndef cmd_display(args):\n return \" \".join(args)\n\ndef run_json(args):\n try:\n out = subprocess.check_output(args, text=True, stderr=subprocess.STDOUT)\n return json.loads(out)\n except subprocess.CalledProcessError as error:\n print(f\"Warning: command failed: {cmd_display(args)}\")\n print(error.output)\n return None\n except json.JSONDecodeError as error:\n print(f\"Warning: non-JSON output from command: {cmd_display(args)} ({error})\")\n return None\n except OSError as error:\n print(f\"Warning: could not execute command: {cmd_display(args)} ({error})\")\n return None\n\ndef run_text(args):\n try:\n return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT)\n except subprocess.CalledProcessError as error:\n print(f\"Warning: command failed: {cmd_display(args)}\")\n print(error.output)\n return \"\"\n except OSError as error:\n print(f\"Warning: could not execute command: {cmd_display(args)} ({error})\")\n return \"\"\n\ndef run_api_json(endpoint, params):\n query = urlencode(params)\n return run_json([\"gh\", \"api\", f\"{endpoint}?{query}\"])\n\ndef is_failure_conclusion(conclusion):\n return (conclusion or \"\").lower() in FAILURE_CONCLUSIONS\n\ndef normalize_workflow_path(path):\n return (path or \"\").split(\"@\", 1)[0]\n\ndef is_agentic_workflow_path(path):\n workflow_path = normalize_workflow_path(path)\n if AGENTIC_WORKFLOW_PATHS:\n return workflow_path in AGENTIC_WORKFLOW_PATHS\n print(\"Warning: no local .lock.yml workflows found; falling back to workflow path suffix matching\")\n return workflow_path.endswith(\".lock.yml\")\n\ndef isoformat_z(dt):\n return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace(\"+00:00\", \"Z\")\n\ndef list_failed_agentic_runs():\n created_since = isoformat_z(datetime.now(timezone.utc) - timedelta(hours=LOOKBACK_HOURS))\n page = 1\n failed_runs = []\n\n while True:\n response = run_api_json(\n f\"repos/{REPO}/actions/runs\",\n {\n \"exclude_pull_requests\": \"true\",\n \"status\": \"completed\",\n \"created\": f\">={created_since}\",\n \"per_page\": \"100\",\n \"page\": str(page),\n },\n ) or {}\n workflow_runs = response.get(\"workflow_runs\") or []\n if not workflow_runs:\n break\n\n for run in workflow_runs:\n workflow_path = normalize_workflow_path(run.get(\"path\"))\n if not is_agentic_workflow_path(workflow_path):\n continue\n if not is_failure_conclusion(run.get(\"conclusion\")):\n continue\n\n failed_runs.append(\n {\n \"run_id\": run.get(\"id\"),\n \"workflow_name\": run.get(\"name\"),\n \"workflow_path\": workflow_path,\n \"created_at\": run.get(\"created_at\"),\n \"status\": run.get(\"status\"),\n \"conclusion\": run.get(\"conclusion\"),\n \"url\": run.get(\"html_url\"),\n }\n )\n\n if len(workflow_runs) < 100:\n break\n if page >= MAX_DISCOVERY_PAGES:\n print(f\"Warning: reached pagination cap ({MAX_DISCOVERY_PAGES} pages) while listing workflow runs\")\n break\n page += 1\n\n failed_runs.sort(key=lambda run: run.get(\"created_at\") or \"\", reverse=True)\n return failed_runs\n\nfailed_runs = list_failed_agentic_runs()\n\n# Cap the number of runs to detail so the payload stays compact.\nfailure_details = []\nfor run in failed_runs[:MAX_FAILURES_TO_DETAIL]:\n run_id = run.get(\"run_id\")\n if not run_id:\n continue\n\n run_view = run_json(\n [\n \"gh\",\n \"run\",\n \"view\",\n str(run_id),\n \"--repo\",\n REPO,\n \"--json\",\n \"databaseId,url,name,workflowName,createdAt,conclusion,status,jobs\",\n ]\n )\n if not run_view:\n continue\n\n failed_job_names = []\n failed_steps = []\n truncated_error_logs = []\n agent_job_conclusion = None\n for job in run_view.get(\"jobs\", []):\n job_name = job.get(\"name\")\n job_conclusion = (job.get(\"conclusion\") or \"\").lower()\n if (job_name or \"\").lower() == \"agent\":\n agent_job_conclusion = job_conclusion or None\n\n if is_failure_conclusion(job_conclusion):\n if job_name:\n failed_job_names.append(job_name)\n for step in job.get(\"steps\", []):\n if is_failure_conclusion(step.get(\"conclusion\")):\n failed_steps.append(\n {\n \"job_id\": job.get(\"databaseId\"),\n \"job_name\": job_name,\n \"step_name\": step.get(\"name\"),\n }\n )\n\n job_id = job.get(\"databaseId\")\n if job_id:\n log_text = run_text(\n [\n \"gh\",\n \"run\",\n \"view\",\n str(run_id),\n \"--repo\",\n REPO,\n \"--job\",\n str(job_id),\n \"--log-failed\",\n ]\n )\n if log_text:\n tail_lines = log_text.splitlines()[-MAX_LOG_TAIL_LINES:]\n truncated_error_logs.append(\n {\n \"job_id\": job_id,\n \"job_name\": job_name,\n \"line_count\": len(tail_lines),\n \"tail_lines\": \"\\n\".join(tail_lines),\n }\n )\n\n failure_details.append(\n {\n \"run_id\": run_id,\n \"workflow_name\": run_view.get(\"workflowName\") or run_view.get(\"name\"),\n \"workflow_path\": run.get(\"workflow_path\"),\n \"url\": run_view.get(\"url\"),\n \"created_at\": run_view.get(\"createdAt\"),\n \"status\": run_view.get(\"status\"),\n \"conclusion\": run_view.get(\"conclusion\"),\n \"failed_job_names\": sorted(set(failed_job_names)),\n \"agent_job_conclusion\": agent_job_conclusion,\n \"failed_steps\": failed_steps,\n \"truncated_error_logs\": truncated_error_logs,\n }\n )\n\nexisting_tracking_issues = run_json(\n [\n \"gh\",\n \"issue\",\n \"list\",\n \"--repo\",\n REPO,\n \"--state\",\n \"open\",\n \"--search\",\n f\"gh-aw-tracker-id: {TRACKER_ID}\",\n \"--limit\",\n \"100\",\n \"--json\",\n \"number,title,state,url,labels,createdAt,updatedAt\",\n ]\n) or []\n\npayload = {\n \"generated_at\": datetime.now(timezone.utc).isoformat(),\n \"repository\": REPO,\n \"lookback_window\": f\"{LOOKBACK_HOURS}h\",\n \"failed_run_ids\": [run.get(\"run_id\") for run in failed_runs if run.get(\"run_id\")],\n \"failures\": failure_details,\n \"existing_tracking_issues\": existing_tracking_issues,\n}\n\nwith open(OUT, \"w\", encoding=\"utf-8\") as f:\n json.dump(payload, f, indent=2)\n f.write(\"\\n\")\n\nprint(f\"Wrote deterministic prefetch payload to {OUT}\")\nprint(f\"Failed runs in payload: {len(payload['failed_run_ids'])}\")\nprint(f\"Existing tracking issues in payload: {len(existing_tracking_issues)}\")\nPY" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index b74e436055c..098a01e8c4d 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -538,6 +538,22 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download CI workflow runs from last 7 days @@ -556,22 +572,6 @@ jobs: name: Pre-flight validation (non-fatal) run: "set +e\nmkdir -p /tmp/gh-aw/agent/validation\nSTATUS_FILE=/tmp/gh-aw/agent/validation/validation-status.json\necho '{\"steps\":[]}' > \"$STATUS_FILE\"\n\nrun_step() {\n local name=\"$1\"; shift\n local workdir=\"$1\"; shift\n local log=\"/tmp/gh-aw/agent/validation/${name}.log\"\n echo \"::group::preflight: $name\"\n echo \"+ $*\" | tee \"$log\"\n if [ -n \"$workdir\" ]; then\n (cd \"$workdir\" && \"$@\") >>\"$log\" 2>&1\n else\n \"$@\" >>\"$log\" 2>&1\n fi\n local code=$?\n echo \"::endgroup::\"\n echo \"preflight: $name exit=$code\"\n # tail kept short so the agent can read all logs cheaply\n tail -c 8000 \"$log\" > \"${log}.tail\"\n jq --arg n \"$name\" --arg log \"$log\" --argjson code \"$code\" \\\n '.steps += [{name:$n, exit_code:$code, log:$log, ok:($code==0)}]' \\\n \"$STATUS_FILE\" > \"$STATUS_FILE.tmp\" && mv \"$STATUS_FILE.tmp\" \"$STATUS_FILE\"\n}\n\nrun_step deps-dev \"\" make deps-dev\nrun_step lint \"\" make lint\nrun_step lint-errors \"\" make lint-errors\nrun_step npm-ci ./actions/setup/js npm ci\nrun_step build \"\" make build\nrun_step recompile \"\" make recompile\n\nmkdir -p /tmp/gh-aw/agent\ngo test -v -json -count=1 -timeout=3m -tags '!integration' -run='^Test' ./... \\\n | tee /tmp/gh-aw/agent/test-results.json >/dev/null\nTEST_EXIT=${PIPESTATUS[0]}\njq --argjson code \"$TEST_EXIT\" \\\n '.steps += [{name:\"test-unit\", exit_code:$code, log:\"/tmp/gh-aw/agent/test-results.json\", ok:($code==0)}]' \\\n \"$STATUS_FILE\" > \"$STATUS_FILE.tmp\" && mv \"$STATUS_FILE.tmp\" \"$STATUS_FILE\"\n\n# Summary line for the agent and step summary\nOK=$(jq '[.steps[] | select(.ok)] | length' \"$STATUS_FILE\")\nTOTAL=$(jq '.steps | length' \"$STATUS_FILE\")\nFAILED=$(jq -r '[.steps[] | select(.ok|not) | .name] | join(\",\")' \"$STATUS_FILE\")\necho \"preflight summary: $OK/$TOTAL ok; failed=[$FAILED]\"\n{\n echo \"## Pre-flight validation\"\n echo \"\"\n echo \"- passed: $OK / $TOTAL\"\n echo \"- failed: ${FAILED:-none}\"\n} >> \"$GITHUB_STEP_SUMMARY\"\n\n# Always succeed: the agent owns the decision about how to react.\nexit 0\n" - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index 0134b1d8c1b..b6eb543e768 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -573,22 +573,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - if: github.event_name == 'workflow_run' - name: Download CI failure logs and artifacts - run: "set -e\nLOG_DIR=\"/tmp/gh-aw/agent/ci-doctor/logs\"\nARTIFACT_DIR=\"/tmp/gh-aw/agent/ci-doctor/artifacts\"\nFILTERED_DIR=\"/tmp/gh-aw/agent/ci-doctor/filtered\"\nmkdir -p \"$LOG_DIR\" \"$ARTIFACT_DIR\" \"$FILTERED_DIR\"\n\necho \"=== CI Doctor: Pre-downloading logs and artifacts for run $RUN_ID ===\"\n\n# Get failed jobs and their failed steps\ngh api \"repos/$REPO/actions/runs/$RUN_ID/jobs\" \\\n --jq '[.jobs[] | select(.conclusion == \"failed\" or .conclusion == \"cancelled\") | {id:.id, name:.name, failed_steps:[.steps[]? | select(.conclusion==\"failed\") | .name]}]' \\\n > \"$LOG_DIR/failed-jobs.json\"\n\nFAILED_COUNT=$(jq 'length' \"$LOG_DIR/failed-jobs.json\")\necho \"Found $FAILED_COUNT failed job(s)\"\n\nif [ \"$FAILED_COUNT\" -eq 0 ]; then\n echo \"No failed jobs found, skipping log download\"\n exit 0\nfi\n\necho \"Failed jobs:\"\ncat \"$LOG_DIR/failed-jobs.json\"\n\n# Download logs for each failed job and apply generic error heuristics\njq -r '.[].id' \"$LOG_DIR/failed-jobs.json\" | while read -r JOB_ID; do\n LOG_FILE=\"$LOG_DIR/job-${JOB_ID}.log\"\n echo \"Downloading log for job $JOB_ID...\"\n gh api \"repos/$REPO/actions/jobs/$JOB_ID/logs\" > \"$LOG_FILE\" 2>/dev/null \\\n || echo \"(log download failed)\" > \"$LOG_FILE\"\n echo \" -> Saved $(wc -l < \"$LOG_FILE\") lines to $LOG_FILE\"\n\n # Apply generic heuristics: find lines with common error indicators\n HINTS_FILE=\"$FILTERED_DIR/job-${JOB_ID}-hints.txt\"\n grep -n -iE \"(error[: ]|ERROR|FAIL|panic:|fatal[: ]|undefined[: ]|exception|exit status [^0])\" \\\n \"$LOG_FILE\" | head -30 > \"$HINTS_FILE\" 2>/dev/null || true\n\n if [ -s \"$HINTS_FILE\" ]; then\n echo \" -> Pre-located $(wc -l < \"$HINTS_FILE\") hint line(s) in $HINTS_FILE\"\n else\n echo \" -> No error hints found in $LOG_FILE\"\n fi\ndone\n\n# Download and unpack all artifacts from the failed run\necho \"\"\necho \"=== Downloading artifacts for run $RUN_ID ===\"\ngh run download \"$RUN_ID\" --repo \"$REPO\" --dir \"$ARTIFACT_DIR\" 2>/dev/null \\\n || echo \"No artifacts available or download failed\"\n\n# Apply heuristics to artifact text files\nfind \"$ARTIFACT_DIR\" -type f \\( \\\n -name \"*.txt\" -o -name \"*.log\" -o -name \"*.json\" \\\n -o -name \"*.xml\" -o -name \"*.out\" -o -name \"*.err\" \\\n\\) | while read -r ARTIFACT_FILE; do\n REL_PATH=\"${ARTIFACT_FILE#\"$ARTIFACT_DIR\"/}\"\n SAFE_NAME=$(echo \"$REL_PATH\" | tr '/' '_')\n HINTS_FILE=\"$FILTERED_DIR/artifact-${SAFE_NAME}-hints.txt\"\n grep -n -iE \"(error[: ]|ERROR|FAIL|panic:|fatal[: ]|undefined[: ]|exception|exit status [^0])\" \\\n \"$ARTIFACT_FILE\" | head -30 > \"$HINTS_FILE\" 2>/dev/null || true\n if [ -s \"$HINTS_FILE\" ]; then\n echo \" -> Artifact hints: $HINTS_FILE ($(wc -l < \"$HINTS_FILE\") lines from $ARTIFACT_FILE)\"\n fi\ndone\n\n# Write summary for the agent\nSUMMARY_FILE=\"/tmp/gh-aw/agent/ci-doctor/summary.txt\"\n{\n echo \"=== CI Doctor Pre-Analysis ===\"\n echo \"Run ID: $RUN_ID\"\n echo \"\"\n echo \"Failed jobs (details in $LOG_DIR/failed-jobs.json):\"\n jq -r '.[] | \" Job \\(.id): \\(.name)\\n Failed steps: \\(.failed_steps | join(\", \"))\"' \\\n \"$LOG_DIR/failed-jobs.json\"\n echo \"\"\n echo \"Downloaded log files ($LOG_DIR):\"\n for LOG_FILE in \"$LOG_DIR\"/job-*.log; do\n [ -f \"$LOG_FILE\" ] || continue\n echo \" $LOG_FILE ($(wc -l < \"$LOG_FILE\") lines)\"\n done\n echo \"\"\n echo \"Downloaded artifact files ($ARTIFACT_DIR):\"\n find \"$ARTIFACT_DIR\" -type f | while read -r f; do\n echo \" $f\"\n done\n echo \"\"\n echo \"Filtered hint files ($FILTERED_DIR):\"\n for HINTS_FILE in \"$FILTERED_DIR\"/*-hints.txt; do\n [ -s \"$HINTS_FILE\" ] || continue\n echo \" $HINTS_FILE ($(wc -l < \"$HINTS_FILE\") matches)\"\n head -3 \"$HINTS_FILE\" | sed 's/^/ /'\n done\n} | tee \"$SUMMARY_FILE\"\n\necho \"\"\necho \"✅ Pre-analysis complete. Agent should start with $SUMMARY_FILE\"\n" - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - PR_NUMBER: ${{ github.event.pull_request.number }} - REPO: ${{ github.repository }} - if: github.event_name == 'pull_request' - name: Fetch PR check run status - run: "set -e\nPR_DIR=\"/tmp/gh-aw/agent/ci-doctor/pr\"\nmkdir -p \"$PR_DIR\"\n\necho \"=== CI Doctor: Fetching check runs for PR #$PR_NUMBER (SHA: $HEAD_SHA) ===\"\n\n# Fetch all check runs for the PR head commit (paginated to handle >30 jobs)\ngh api --paginate \"repos/$REPO/commits/$HEAD_SHA/check-runs\" \\\n --jq '.check_runs[] | {id:.id, name:.name, status:.status, conclusion:.conclusion, html_url:.html_url}' \\\n | jq -s '.' \\\n > \"$PR_DIR/check-runs.json\"\n\nTOTAL=$(jq 'length' \"$PR_DIR/check-runs.json\")\nFAILED=$(jq '[.[] | select(.conclusion == \"failure\" or .conclusion == \"cancelled\" or .conclusion == \"timed_out\")] | length' \"$PR_DIR/check-runs.json\")\necho \"Found $TOTAL check run(s), $FAILED failing\"\n\n# Isolate the failing check runs\njq '[.[] | select(.conclusion == \"failure\" or .conclusion == \"cancelled\" or .conclusion == \"timed_out\")]' \\\n \"$PR_DIR/check-runs.json\" > \"$PR_DIR/failed-checks.json\"\n\n# Write a human-readable summary\nSUMMARY_FILE=\"$PR_DIR/summary.txt\"\n{\n echo \"=== CI Doctor PR Pre-Analysis ===\"\n echo \"PR: #$PR_NUMBER\"\n echo \"HEAD SHA: $HEAD_SHA\"\n echo \"Total check runs: $TOTAL\"\n echo \"Failing check runs: $FAILED\"\n echo \"\"\n echo \"All checks ($PR_DIR/check-runs.json):\"\n jq -r '.[] | \" \\(.conclusion // .status): \\(.name)\"' \"$PR_DIR/check-runs.json\"\n echo \"\"\n if [ \"$FAILED\" -gt 0 ]; then\n echo \"Failing checks ($PR_DIR/failed-checks.json):\"\n jq -r '.[] | \" - \\(.name) [\\(.conclusion)]: \\(.html_url)\"' \"$PR_DIR/failed-checks.json\"\n fi\n} | tee \"$SUMMARY_FILE\"\n\necho \"\"\necho \"✅ PR pre-analysis complete. Agent should start with $SUMMARY_FILE\"" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -605,6 +589,22 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + if: github.event_name == 'workflow_run' + name: Download CI failure logs and artifacts + run: "set -e\nLOG_DIR=\"/tmp/gh-aw/agent/ci-doctor/logs\"\nARTIFACT_DIR=\"/tmp/gh-aw/agent/ci-doctor/artifacts\"\nFILTERED_DIR=\"/tmp/gh-aw/agent/ci-doctor/filtered\"\nmkdir -p \"$LOG_DIR\" \"$ARTIFACT_DIR\" \"$FILTERED_DIR\"\n\necho \"=== CI Doctor: Pre-downloading logs and artifacts for run $RUN_ID ===\"\n\n# Get failed jobs and their failed steps\ngh api \"repos/$REPO/actions/runs/$RUN_ID/jobs\" \\\n --jq '[.jobs[] | select(.conclusion == \"failed\" or .conclusion == \"cancelled\") | {id:.id, name:.name, failed_steps:[.steps[]? | select(.conclusion==\"failed\") | .name]}]' \\\n > \"$LOG_DIR/failed-jobs.json\"\n\nFAILED_COUNT=$(jq 'length' \"$LOG_DIR/failed-jobs.json\")\necho \"Found $FAILED_COUNT failed job(s)\"\n\nif [ \"$FAILED_COUNT\" -eq 0 ]; then\n echo \"No failed jobs found, skipping log download\"\n exit 0\nfi\n\necho \"Failed jobs:\"\ncat \"$LOG_DIR/failed-jobs.json\"\n\n# Download logs for each failed job and apply generic error heuristics\njq -r '.[].id' \"$LOG_DIR/failed-jobs.json\" | while read -r JOB_ID; do\n LOG_FILE=\"$LOG_DIR/job-${JOB_ID}.log\"\n echo \"Downloading log for job $JOB_ID...\"\n gh api \"repos/$REPO/actions/jobs/$JOB_ID/logs\" > \"$LOG_FILE\" 2>/dev/null \\\n || echo \"(log download failed)\" > \"$LOG_FILE\"\n echo \" -> Saved $(wc -l < \"$LOG_FILE\") lines to $LOG_FILE\"\n\n # Apply generic heuristics: find lines with common error indicators\n HINTS_FILE=\"$FILTERED_DIR/job-${JOB_ID}-hints.txt\"\n grep -n -iE \"(error[: ]|ERROR|FAIL|panic:|fatal[: ]|undefined[: ]|exception|exit status [^0])\" \\\n \"$LOG_FILE\" | head -30 > \"$HINTS_FILE\" 2>/dev/null || true\n\n if [ -s \"$HINTS_FILE\" ]; then\n echo \" -> Pre-located $(wc -l < \"$HINTS_FILE\") hint line(s) in $HINTS_FILE\"\n else\n echo \" -> No error hints found in $LOG_FILE\"\n fi\ndone\n\n# Download and unpack all artifacts from the failed run\necho \"\"\necho \"=== Downloading artifacts for run $RUN_ID ===\"\ngh run download \"$RUN_ID\" --repo \"$REPO\" --dir \"$ARTIFACT_DIR\" 2>/dev/null \\\n || echo \"No artifacts available or download failed\"\n\n# Apply heuristics to artifact text files\nfind \"$ARTIFACT_DIR\" -type f \\( \\\n -name \"*.txt\" -o -name \"*.log\" -o -name \"*.json\" \\\n -o -name \"*.xml\" -o -name \"*.out\" -o -name \"*.err\" \\\n\\) | while read -r ARTIFACT_FILE; do\n REL_PATH=\"${ARTIFACT_FILE#\"$ARTIFACT_DIR\"/}\"\n SAFE_NAME=$(echo \"$REL_PATH\" | tr '/' '_')\n HINTS_FILE=\"$FILTERED_DIR/artifact-${SAFE_NAME}-hints.txt\"\n grep -n -iE \"(error[: ]|ERROR|FAIL|panic:|fatal[: ]|undefined[: ]|exception|exit status [^0])\" \\\n \"$ARTIFACT_FILE\" | head -30 > \"$HINTS_FILE\" 2>/dev/null || true\n if [ -s \"$HINTS_FILE\" ]; then\n echo \" -> Artifact hints: $HINTS_FILE ($(wc -l < \"$HINTS_FILE\") lines from $ARTIFACT_FILE)\"\n fi\ndone\n\n# Write summary for the agent\nSUMMARY_FILE=\"/tmp/gh-aw/agent/ci-doctor/summary.txt\"\n{\n echo \"=== CI Doctor Pre-Analysis ===\"\n echo \"Run ID: $RUN_ID\"\n echo \"\"\n echo \"Failed jobs (details in $LOG_DIR/failed-jobs.json):\"\n jq -r '.[] | \" Job \\(.id): \\(.name)\\n Failed steps: \\(.failed_steps | join(\", \"))\"' \\\n \"$LOG_DIR/failed-jobs.json\"\n echo \"\"\n echo \"Downloaded log files ($LOG_DIR):\"\n for LOG_FILE in \"$LOG_DIR\"/job-*.log; do\n [ -f \"$LOG_FILE\" ] || continue\n echo \" $LOG_FILE ($(wc -l < \"$LOG_FILE\") lines)\"\n done\n echo \"\"\n echo \"Downloaded artifact files ($ARTIFACT_DIR):\"\n find \"$ARTIFACT_DIR\" -type f | while read -r f; do\n echo \" $f\"\n done\n echo \"\"\n echo \"Filtered hint files ($FILTERED_DIR):\"\n for HINTS_FILE in \"$FILTERED_DIR\"/*-hints.txt; do\n [ -s \"$HINTS_FILE\" ] || continue\n echo \" $HINTS_FILE ($(wc -l < \"$HINTS_FILE\") matches)\"\n head -3 \"$HINTS_FILE\" | sed 's/^/ /'\n done\n} | tee \"$SUMMARY_FILE\"\n\necho \"\"\necho \"✅ Pre-analysis complete. Agent should start with $SUMMARY_FILE\"\n" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + if: github.event_name == 'pull_request' + name: Fetch PR check run status + run: "set -e\nPR_DIR=\"/tmp/gh-aw/agent/ci-doctor/pr\"\nmkdir -p \"$PR_DIR\"\n\necho \"=== CI Doctor: Fetching check runs for PR #$PR_NUMBER (SHA: $HEAD_SHA) ===\"\n\n# Fetch all check runs for the PR head commit (paginated to handle >30 jobs)\ngh api --paginate \"repos/$REPO/commits/$HEAD_SHA/check-runs\" \\\n --jq '.check_runs[] | {id:.id, name:.name, status:.status, conclusion:.conclusion, html_url:.html_url}' \\\n | jq -s '.' \\\n > \"$PR_DIR/check-runs.json\"\n\nTOTAL=$(jq 'length' \"$PR_DIR/check-runs.json\")\nFAILED=$(jq '[.[] | select(.conclusion == \"failure\" or .conclusion == \"cancelled\" or .conclusion == \"timed_out\")] | length' \"$PR_DIR/check-runs.json\")\necho \"Found $TOTAL check run(s), $FAILED failing\"\n\n# Isolate the failing check runs\njq '[.[] | select(.conclusion == \"failure\" or .conclusion == \"cancelled\" or .conclusion == \"timed_out\")]' \\\n \"$PR_DIR/check-runs.json\" > \"$PR_DIR/failed-checks.json\"\n\n# Write a human-readable summary\nSUMMARY_FILE=\"$PR_DIR/summary.txt\"\n{\n echo \"=== CI Doctor PR Pre-Analysis ===\"\n echo \"PR: #$PR_NUMBER\"\n echo \"HEAD SHA: $HEAD_SHA\"\n echo \"Total check runs: $TOTAL\"\n echo \"Failing check runs: $FAILED\"\n echo \"\"\n echo \"All checks ($PR_DIR/check-runs.json):\"\n jq -r '.[] | \" \\(.conclusion // .status): \\(.name)\"' \"$PR_DIR/check-runs.json\"\n echo \"\"\n if [ \"$FAILED\" -gt 0 ]; then\n echo \"Failing checks ($PR_DIR/failed-checks.json):\"\n jq -r '.[] | \" - \\(.name) [\\(.conclusion)]: \\(.html_url)\"' \"$PR_DIR/failed-checks.json\"\n fi\n} | tee \"$SUMMARY_FILE\"\n\necho \"\"\necho \"✅ PR pre-analysis complete. Agent should start with $SUMMARY_FILE\"" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-agent-analysis.lock.yml b/.github/workflows/copilot-agent-analysis.lock.yml index 5cbbf290fc9..a4455ab874d 100644 --- a/.github/workflows/copilot-agent-analysis.lock.yml +++ b/.github/workflows/copilot-agent-analysis.lock.yml @@ -544,15 +544,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install gh CLI - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Fetch Copilot PR data - run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -579,6 +570,15 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Install gh CLI + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Fetch Copilot PR data + run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-centralization-optimizer.lock.yml b/.github/workflows/copilot-centralization-optimizer.lock.yml index 12667f9ea02..a10cc79fcd1 100644 --- a/.github/workflows/copilot-centralization-optimizer.lock.yml +++ b/.github/workflows/copilot-centralization-optimizer.lock.yml @@ -459,6 +459,16 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone repo-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: memory/copilot-centralization-optimizer + TARGET_REPO: ${{ github.repository }} + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - env: GH_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} REPO: ${{ github.repository }} @@ -483,16 +493,6 @@ jobs: path: /tmp/gh-aw/data retention-days: 14 - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone repo-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: memory/copilot-centralization-optimizer - TARGET_REPO: ${{ github.repository }} - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-opt.lock.yml b/.github/workflows/copilot-opt.lock.yml index a2676186633..b4d2b437721 100644 --- a/.github/workflows/copilot-opt.lock.yml +++ b/.github/workflows/copilot-opt.lock.yml @@ -487,6 +487,22 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-copilot-session-data-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-copilot-session-data- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" @@ -504,22 +520,6 @@ jobs: name: Fetch Copilot PR data run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-copilot-session-data-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-copilot-session-data- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-pr-merged-report.lock.yml b/.github/workflows/copilot-pr-merged-report.lock.yml index ffadf0bbcc3..5cdd3b6fa03 100644 --- a/.github/workflows/copilot-pr-merged-report.lock.yml +++ b/.github/workflows/copilot-pr-merged-report.lock.yml @@ -438,15 +438,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install gh CLI - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Fetch Copilot PR data - run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -463,6 +454,15 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Install gh CLI + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Fetch Copilot PR data + run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-pr-nlp-analysis.lock.yml b/.github/workflows/copilot-pr-nlp-analysis.lock.yml index 88d3ab5d633..8f92ec84121 100644 --- a/.github/workflows/copilot-pr-nlp-analysis.lock.yml +++ b/.github/workflows/copilot-pr-nlp-analysis.lock.yml @@ -508,6 +508,32 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone repo-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: memory/nlp-analysis + TARGET_REPO: ${{ github.repository }} + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries @@ -538,32 +564,6 @@ jobs: name: Fetch PR comments for detailed analysis run: "# Create comments directory\nmkdir -p /tmp/gh-aw/agent/pr-comments\n\n# Fetch detailed comments for each PR from the pre-fetched data\nPR_COUNT=$(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\necho \"Fetching comments for $PR_COUNT PRs...\"\n\njq -r '.[].number' /tmp/gh-aw/agent/pr-data/copilot-prs.json | while read -r PR_NUM; do\n echo \"Fetching comments for PR #${PR_NUM}\"\n gh pr view \"${PR_NUM}\" \\\n --json comments,reviews,reviewComments \\\n > \"/tmp/gh-aw/agent/pr-comments/pr-${PR_NUM}.json\" 2>/dev/null || echo \"{}\" > \"/tmp/gh-aw/agent/pr-comments/pr-${PR_NUM}.json\"\n sleep 0.5 # Rate limiting\ndone\n\necho \"Comment data saved to /tmp/gh-aw/agent/pr-comments/\"" - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone repo-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: memory/nlp-analysis - TARGET_REPO: ${{ github.repository }} - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-pr-prompt-analysis.lock.yml b/.github/workflows/copilot-pr-prompt-analysis.lock.yml index ddb45ce75db..7ba36ce5d67 100644 --- a/.github/workflows/copilot-pr-prompt-analysis.lock.yml +++ b/.github/workflows/copilot-pr-prompt-analysis.lock.yml @@ -497,15 +497,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install gh CLI - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Fetch Copilot PR data - run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -532,6 +523,15 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Install gh CLI + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Fetch Copilot PR data + run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/pr-data\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-prs-${TODAY}.json\" ]; then\n echo \"✓ Found cached PR data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-prs-${TODAY}.json\" /tmp/gh-aw/agent/pr-data/copilot-prs.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\" /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total PRs in cache: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nelse\n echo \"⬇ Downloading fresh PR data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for PRs from copilot/* branches in the last 30 days using gh CLI\n # Using branch prefix search (head:copilot/) instead of author for reliability\n echo \"Fetching Copilot PRs from the last 30 days...\"\n gh pr list --repo \"$GITHUB_REPOSITORY\" \\\n --search \"head:copilot/ created:>=${DATE_30_DAYS_AGO}\" \\\n --state all \\\n --json number,title,author,headRefName,createdAt,state,url,body,labels,updatedAt,closedAt,mergedAt \\\n --limit 1000 \\\n > /tmp/gh-aw/agent/pr-data/copilot-prs.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/pr-data/copilot-prs.json > /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/pr-data/copilot-prs.json \"$CACHE_DIR/copilot-prs-${TODAY}.json\"\n cp /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json \"$CACHE_DIR/copilot-prs-${TODAY}-schema.json\"\n\n echo \"✓ PR data saved to cache: copilot-prs-${TODAY}.json\"\n echo \"Total PRs found: $(jq 'length' /tmp/gh-aw/agent/pr-data/copilot-prs.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"PR data available at: /tmp/gh-aw/agent/pr-data/copilot-prs.json\"\necho \"Schema available at: /tmp/gh-aw/agent/pr-data/copilot-prs-schema.json\"\n" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/copilot-session-insights.lock.yml b/.github/workflows/copilot-session-insights.lock.yml index 026165d858e..4c37f9b26ea 100644 --- a/.github/workflows/copilot-session-insights.lock.yml +++ b/.github/workflows/copilot-session-insights.lock.yml @@ -522,29 +522,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install gh CLI - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Fetch Copilot session data - run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/session-data\nmkdir -p /tmp/gh-aw/agent/session-data/logs\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-sessions-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-sessions-${TODAY}.json\" ]; then\n echo \"✓ Found cached session data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-sessions-${TODAY}.json\" /tmp/gh-aw/agent/session-data/sessions-list.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/session-data/sessions-list.json > \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\" /tmp/gh-aw/agent/session-data/sessions-schema.json\n \n # Restore cached log files if they exist\n if [ -d \"$CACHE_DIR/session-logs-${TODAY}\" ]; then\n echo \"✓ Found cached session logs from ${TODAY}\"\n cp -r \"$CACHE_DIR/session-logs-${TODAY}\"/* /tmp/gh-aw/agent/session-data/logs/ 2>/dev/null || true\n echo \"Restored $(find /tmp/gh-aw/agent/session-data/logs -type f | wc -l) session log files from cache\"\n fi\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total sessions in cache: $(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\"\nelse\n echo \"⬇ Downloading fresh session data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for workflow runs from copilot/* branches\n # This fetches GitHub Copilot coding agent task runs by searching for workflow runs on copilot/* branches\n echo \"Fetching Copilot coding agent workflow runs from the last 30 days...\"\n \n # Get workflow runs from copilot/* branches\n gh api \"repos/$GITHUB_REPOSITORY/actions/runs\" \\\n --paginate \\\n --jq \".workflow_runs[] | select(.head_branch | startswith(\\\"copilot/\\\")) | select(.created_at >= \\\"${DATE_30_DAYS_AGO}\\\") | {id, name, head_branch, created_at, updated_at, status, conclusion, html_url}\" \\\n | jq -s '.[0:50]' \\\n > /tmp/gh-aw/agent/session-data/sessions-list.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/session-data/sessions-list.json > /tmp/gh-aw/agent/session-data/sessions-schema.json\n\n # Download conversation logs using gh agent-task command (limit to first 50)\n SESSION_COUNT=$(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\n echo \"Downloading conversation logs for $SESSION_COUNT sessions...\"\n \n # Use gh agent-task to fetch session logs with conversation transcripts\n # Extract session numbers from head_branch (format: copilot/issue-123 or copilot/task-456)\n # The number is the issue/task/PR number that the gh agent-task command uses\n jq -r '.[].head_branch' /tmp/gh-aw/agent/session-data/sessions-list.json | while read -r branch; do\n if [ -n \"$branch\" ]; then\n # Extract number from branch name (e.g., copilot/issue-123 -> 123)\n # This is the session identifier used by gh agent-task\n session_number=$(echo \"$branch\" | sed 's/copilot\\///' | sed 's/[^0-9]//g')\n \n if [ -n \"$session_number\" ]; then\n echo \"Downloading conversation log for session #$session_number (branch: $branch)\"\n \n # Use gh agent-task view --log to get conversation transcript\n # This contains the agent's internal monologue, tool calls, and reasoning\n gh agent-task view --repo \"$GITHUB_REPOSITORY\" \"$session_number\" --log \\\n > \"/tmp/gh-aw/agent/session-data/logs/${session_number}-conversation.txt\" 2>&1 || {\n echo \"Warning: Could not fetch conversation log for session #$session_number\"\n # If gh agent-task fails, fall back to downloading GitHub Actions logs\n # This ensures we have some data even if agent-task command is unavailable\n run_id=$(jq -r \".[] | select(.head_branch == \\\"$branch\\\") | .id\" /tmp/gh-aw/agent/session-data/sessions-list.json)\n if [ -n \"$run_id\" ]; then\n echo \"Falling back to GitHub Actions logs for run ID: $run_id\"\n gh api \"repos/$GITHUB_REPOSITORY/actions/runs/${run_id}/logs\" \\\n > \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" 2>&1 || true\n \n if [ -f \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" ] && [ -s \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" ]; then\n unzip -q \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" -d \"/tmp/gh-aw/agent/session-data/logs/${session_number}/\" 2>/dev/null || true\n rm \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\"\n fi\n fi\n }\n fi\n fi\n done\n \n LOG_COUNT=$(find /tmp/gh-aw/agent/session-data/logs/ -type f -name \"*-conversation.txt\" | wc -l)\n echo \"Conversation logs downloaded: $LOG_COUNT session logs\"\n \n FALLBACK_COUNT=$(find /tmp/gh-aw/agent/session-data/logs/ -type d -mindepth 1 | wc -l)\n if [ \"$FALLBACK_COUNT\" -gt 0 ]; then\n echo \"Fallback GitHub Actions logs: $FALLBACK_COUNT sessions\"\n fi\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/session-data/sessions-list.json \"$CACHE_DIR/copilot-sessions-${TODAY}.json\"\n cp /tmp/gh-aw/agent/session-data/sessions-schema.json \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\"\n \n # Cache the log files\n mkdir -p \"$CACHE_DIR/session-logs-${TODAY}\"\n cp -r /tmp/gh-aw/agent/session-data/logs/* \"$CACHE_DIR/session-logs-${TODAY}/\" 2>/dev/null || true\n\n echo \"✓ Session data saved to cache: copilot-sessions-${TODAY}.json\"\n echo \"Total sessions found: $(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"Session data available at: /tmp/gh-aw/agent/session-data/sessions-list.json\"\necho \"Schema available at: /tmp/gh-aw/agent/session-data/sessions-schema.json\"\necho \"Logs available at: /tmp/gh-aw/agent/session-data/logs/\"\n\n# Set outputs for downstream use\necho \"sessions_count=$(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\" >> \"$GITHUB_OUTPUT\"\n" - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -571,6 +548,29 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Install gh CLI + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Fetch Copilot session data + run: "# Create output directories\nmkdir -p /tmp/gh-aw/agent/session-data\nmkdir -p /tmp/gh-aw/agent/session-data/logs\nmkdir -p /tmp/gh-aw/cache-memory\n\n# Get today's date for cache identification\nTODAY=$(date '+%Y-%m-%d')\nCACHE_DIR=\"/tmp/gh-aw/cache-memory\"\n\n# Check if cached data exists from today\nif [ -f \"$CACHE_DIR/copilot-sessions-${TODAY}.json\" ] && [ -s \"$CACHE_DIR/copilot-sessions-${TODAY}.json\" ]; then\n echo \"✓ Found cached session data from ${TODAY}\"\n cp \"$CACHE_DIR/copilot-sessions-${TODAY}.json\" /tmp/gh-aw/agent/session-data/sessions-list.json\n \n # Regenerate schema if missing\n if [ ! -f \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\" ]; then\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/session-data/sessions-list.json > \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\"\n fi\n cp \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\" /tmp/gh-aw/agent/session-data/sessions-schema.json\n \n # Restore cached log files if they exist\n if [ -d \"$CACHE_DIR/session-logs-${TODAY}\" ]; then\n echo \"✓ Found cached session logs from ${TODAY}\"\n cp -r \"$CACHE_DIR/session-logs-${TODAY}\"/* /tmp/gh-aw/agent/session-data/logs/ 2>/dev/null || true\n echo \"Restored $(find /tmp/gh-aw/agent/session-data/logs -type f | wc -l) session log files from cache\"\n fi\n \n echo \"Using cached data from ${TODAY}\"\n echo \"Total sessions in cache: $(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\"\nelse\n echo \"⬇ Downloading fresh session data...\"\n \n # Calculate date 30 days ago\n DATE_30_DAYS_AGO=$(date -d '30 days ago' '+%Y-%m-%d' 2>/dev/null || date -v-30d '+%Y-%m-%d')\n\n # Search for workflow runs from copilot/* branches\n # This fetches GitHub Copilot coding agent task runs by searching for workflow runs on copilot/* branches\n echo \"Fetching Copilot coding agent workflow runs from the last 30 days...\"\n \n # Get workflow runs from copilot/* branches\n gh api \"repos/$GITHUB_REPOSITORY/actions/runs\" \\\n --paginate \\\n --jq \".workflow_runs[] | select(.head_branch | startswith(\\\"copilot/\\\")) | select(.created_at >= \\\"${DATE_30_DAYS_AGO}\\\") | {id, name, head_branch, created_at, updated_at, status, conclusion, html_url}\" \\\n | jq -s '.[0:50]' \\\n > /tmp/gh-aw/agent/session-data/sessions-list.json\n\n # Generate schema for reference\n ./.github/skills/jqschema/jqschema.sh < /tmp/gh-aw/agent/session-data/sessions-list.json > /tmp/gh-aw/agent/session-data/sessions-schema.json\n\n # Download conversation logs using gh agent-task command (limit to first 50)\n SESSION_COUNT=$(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\n echo \"Downloading conversation logs for $SESSION_COUNT sessions...\"\n \n # Use gh agent-task to fetch session logs with conversation transcripts\n # Extract session numbers from head_branch (format: copilot/issue-123 or copilot/task-456)\n # The number is the issue/task/PR number that the gh agent-task command uses\n jq -r '.[].head_branch' /tmp/gh-aw/agent/session-data/sessions-list.json | while read -r branch; do\n if [ -n \"$branch\" ]; then\n # Extract number from branch name (e.g., copilot/issue-123 -> 123)\n # This is the session identifier used by gh agent-task\n session_number=$(echo \"$branch\" | sed 's/copilot\\///' | sed 's/[^0-9]//g')\n \n if [ -n \"$session_number\" ]; then\n echo \"Downloading conversation log for session #$session_number (branch: $branch)\"\n \n # Use gh agent-task view --log to get conversation transcript\n # This contains the agent's internal monologue, tool calls, and reasoning\n gh agent-task view --repo \"$GITHUB_REPOSITORY\" \"$session_number\" --log \\\n > \"/tmp/gh-aw/agent/session-data/logs/${session_number}-conversation.txt\" 2>&1 || {\n echo \"Warning: Could not fetch conversation log for session #$session_number\"\n # If gh agent-task fails, fall back to downloading GitHub Actions logs\n # This ensures we have some data even if agent-task command is unavailable\n run_id=$(jq -r \".[] | select(.head_branch == \\\"$branch\\\") | .id\" /tmp/gh-aw/agent/session-data/sessions-list.json)\n if [ -n \"$run_id\" ]; then\n echo \"Falling back to GitHub Actions logs for run ID: $run_id\"\n gh api \"repos/$GITHUB_REPOSITORY/actions/runs/${run_id}/logs\" \\\n > \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" 2>&1 || true\n \n if [ -f \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" ] && [ -s \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" ]; then\n unzip -q \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\" -d \"/tmp/gh-aw/agent/session-data/logs/${session_number}/\" 2>/dev/null || true\n rm \"/tmp/gh-aw/agent/session-data/logs/${session_number}-actions.zip\"\n fi\n fi\n }\n fi\n fi\n done\n \n LOG_COUNT=$(find /tmp/gh-aw/agent/session-data/logs/ -type f -name \"*-conversation.txt\" | wc -l)\n echo \"Conversation logs downloaded: $LOG_COUNT session logs\"\n \n FALLBACK_COUNT=$(find /tmp/gh-aw/agent/session-data/logs/ -type d -mindepth 1 | wc -l)\n if [ \"$FALLBACK_COUNT\" -gt 0 ]; then\n echo \"Fallback GitHub Actions logs: $FALLBACK_COUNT sessions\"\n fi\n\n # Store in cache with today's date\n cp /tmp/gh-aw/agent/session-data/sessions-list.json \"$CACHE_DIR/copilot-sessions-${TODAY}.json\"\n cp /tmp/gh-aw/agent/session-data/sessions-schema.json \"$CACHE_DIR/copilot-sessions-${TODAY}-schema.json\"\n \n # Cache the log files\n mkdir -p \"$CACHE_DIR/session-logs-${TODAY}\"\n cp -r /tmp/gh-aw/agent/session-data/logs/* \"$CACHE_DIR/session-logs-${TODAY}/\" 2>/dev/null || true\n\n echo \"✓ Session data saved to cache: copilot-sessions-${TODAY}.json\"\n echo \"Total sessions found: $(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\"\nfi\n\n# Always ensure data is available at expected locations for backward compatibility\necho \"Session data available at: /tmp/gh-aw/agent/session-data/sessions-list.json\"\necho \"Schema available at: /tmp/gh-aw/agent/session-data/sessions-schema.json\"\necho \"Logs available at: /tmp/gh-aw/agent/session-data/logs/\"\n\n# Set outputs for downstream use\necho \"sessions_count=$(jq 'length' /tmp/gh-aw/agent/session-data/sessions-list.json)\" >> \"$GITHUB_OUTPUT\"\n" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-code-metrics.lock.yml b/.github/workflows/daily-code-metrics.lock.yml index c4d474bf9e2..1dcaf3593a1 100644 --- a/.github/workflows/daily-code-metrics.lock.yml +++ b/.github/workflows/daily-code-metrics.lock.yml @@ -548,21 +548,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -589,6 +574,21 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-community-attribution.lock.yml b/.github/workflows/daily-community-attribution.lock.yml index 5d8b536d1d9..3f8e3d52539 100644 --- a/.github/workflows/daily-community-attribution.lock.yml +++ b/.github/workflows/daily-community-attribution.lock.yml @@ -530,6 +530,16 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone wiki-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: master + TARGET_REPO: ${{ github.repository }}.wiki + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: false + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -548,16 +558,6 @@ jobs: - name: Format attribution data for agent run: "DATA_DIR=/tmp/gh-aw/agent/community-data\n\n# Group Tier 0-2 attributions by author for agent-ready consumption.\n# Produces a structured JSON the agent can read with `cat` — no jq needed.\njq '\n group_by(.author.login) |\n sort_by(.[0].author.login | ascii_downcase) |\n map({\n author: .[0].author.login,\n count: length,\n issues: (sort_by(-.number))\n })\n' \"$DATA_DIR/pre_attributed.json\" \\\n > \"$DATA_DIR/attribution_by_author.json\" 2>/dev/null \\\n || echo \"[]\" > \"$DATA_DIR/attribution_by_author.json\"\n\nAUTHOR_COUNT=$(jq length \"$DATA_DIR/attribution_by_author.json\")\nISSUE_COUNT=$(jq length \"$DATA_DIR/pre_attributed.json\")\necho \"✓ Grouped: $AUTHOR_COUNT authors, $ISSUE_COUNT issues\"\n\n# Generate the pre-formatted README community section (Tier 0-2 only).\n# The agent reads this file directly, appends Tier 3 results, and inserts\n# the result into README.md — no bash data-processing required.\n{\n REPO_URL=\"https://github.com/${GITHUB_REPOSITORY:-github/gh-aw}\"\n SEARCH_BASE=\"${REPO_URL}/issues?q=is%3Aissue+is%3Aclosed+label%3Acommunity+author%3A\"\n echo \"## 🌍 Community Contributions\"\n echo \"\"\n echo \"Community members whose issues were resolved — updated automatically.\"\n echo \"\"\n jq -r --arg base \"$SEARCH_BASE\" '\n .[] |\n \"[@\\(.author) (\\(.count))](\\($base + .author))\"\n ' \"$DATA_DIR/attribution_by_author.json\"\n echo \"\"\n} > \"$DATA_DIR/readme_community_section_tier012.md\"\n\necho \"✓ Generated readme_community_section_tier012.md\"\necho \"\"\necho \"Data available in $DATA_DIR/:\"\necho \" attribution_by_author.json — Tier 0-2 issues grouped by author (agent-ready)\"\necho \" readme_community_section_tier012.md — pre-formatted README section (Tier 0-2 only)\"" - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone wiki-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: master - TARGET_REPO: ${{ github.repository }}.wiki - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: false - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-experiment-report.lock.yml b/.github/workflows/daily-experiment-report.lock.yml index 75ec7c19007..21f43deed59 100644 --- a/.github/workflows/daily-experiment-report.lock.yml +++ b/.github/workflows/daily-experiment-report.lock.yml @@ -489,6 +489,22 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Setup Python environment run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" - if: always() @@ -507,22 +523,6 @@ jobs: pypy3 -m pip install --quiet numpy matplotlib fi - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index 67b2f9d5756..70aec853d44 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -637,11 +637,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install MemPalace - run: pip install "mempalace==3.2.0" - - name: Start MemPalace MCP Server - run: "set -e\nmkdir -p /tmp/gh-aw/mcp-logs/mempalace\nmkdir -p /tmp/gh-aw/cache-memory/palace\npython -m mempalace.mcp_server \\\n --palace /tmp/gh-aw/cache-memory/palace \\\n --transport streamable-http \\\n --host 127.0.0.1 \\\n --port 8765 \\\n > /tmp/gh-aw/mcp-logs/mempalace/server.log 2>&1 &\necho \"MemPalace MCP server started (PID $!)\"\n" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -658,6 +653,11 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Install MemPalace + run: pip install "mempalace==3.2.0" + - name: Start MemPalace MCP Server + run: "set -e\nmkdir -p /tmp/gh-aw/mcp-logs/mempalace\nmkdir -p /tmp/gh-aw/cache-memory/palace\npython -m mempalace.mcp_server \\\n --palace /tmp/gh-aw/cache-memory/palace \\\n --transport streamable-http \\\n --host 127.0.0.1 \\\n --port 8765 \\\n > /tmp/gh-aw/mcp-logs/mempalace/server.log 2>&1 &\necho \"MemPalace MCP server started (PID $!)\"\n" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-hippo-learn.lock.yml b/.github/workflows/daily-hippo-learn.lock.yml index 9e36db8c81d..19c65f7a2c7 100644 --- a/.github/workflows/daily-hippo-learn.lock.yml +++ b/.github/workflows/daily-hippo-learn.lock.yml @@ -480,12 +480,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install hippo-memory - run: | - npm install -g hippo-memory - - name: Initialize hippo store - run: "# Symlink .hippo into cache-memory so the SQLite store persists across runs.\n# All writes to .hippo/ land in /tmp/gh-aw/cache-memory/hippo-store/ and are\n# saved/restored automatically by the cache-memory mechanism.\nmkdir -p /tmp/gh-aw/cache-memory/hippo-store\n\nif [ ! -e \".hippo\" ]; then\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Created .hippo → cache-memory/hippo-store\"\nelif [ -d \".hippo\" ] && [ ! -L \".hippo\" ]; then\n # Plain directory present (e.g. first run after adding this import) — migrate\n cp -r .hippo/. /tmp/gh-aw/cache-memory/hippo-store/ 2>/dev/null || true\n rm -rf .hippo\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Migrated existing .hippo/ → cache-memory/hippo-store\"\nelse\n echo \"✅ .hippo already linked to cache-memory/hippo-store\"\nfi\n\n# Initialise if the store has never been set up. Perform a one-time\n# repository scan to seed the memory store with historical incidents.\n# Set HIPPO_SCAN_DAYS to override the default 365-day scan window.\nHIPPO_SCAN_DAYS=\"${HIPPO_SCAN_DAYS:-365}\"\nif [ ! -f \".hippo/config.json\" ]; then\n hippo init --scan --days \"$HIPPO_SCAN_DAYS\"\n echo \"✅ Hippo memory store initialised (scan days: $HIPPO_SCAN_DAYS)\"\nelse\n echo \"✅ Hippo store restored from cache\"\n hippo list 2>/dev/null | head -5 || true\nfi\n\n# One-time bootstrap for repositories with an empty or freshly initialised\n# store. This seeds memory with key project docs and recurring incidents.\nif [ ! -f \".hippo/.gh-aw-bootstrap-v1\" ]; then\n if [ -f \"AGENTS.md\" ]; then\n hippo import --markdown AGENTS.md\n fi\n\n INCIDENTS=(\n 'Recurring incident: Codex auth failures can break agent jobs; verify auth/mode/token setup before reruns.'\n 'Recurring incident: stale workflow .lock.yml files cause churn and CI friction; run make recompile after markdown workflow edits.'\n 'Recurring incident: node: command not found on GPU/self-hosted paths when node runtime/tooling is missing; validate runtimes and PATH early.'\n )\n for incident in \"${INCIDENTS[@]}\"; do\n hippo remember \"$incident\" --tag incident\n done\n\n touch \".hippo/.gh-aw-bootstrap-v1\"\n echo \"✅ Hippo store bootstrap complete (.gh-aw-bootstrap-v1)\"\nfi\n" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -502,6 +496,12 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Install hippo-memory + run: | + npm install -g hippo-memory + - name: Initialize hippo store + run: "# Symlink .hippo into cache-memory so the SQLite store persists across runs.\n# All writes to .hippo/ land in /tmp/gh-aw/cache-memory/hippo-store/ and are\n# saved/restored automatically by the cache-memory mechanism.\nmkdir -p /tmp/gh-aw/cache-memory/hippo-store\n\nif [ ! -e \".hippo\" ]; then\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Created .hippo → cache-memory/hippo-store\"\nelif [ -d \".hippo\" ] && [ ! -L \".hippo\" ]; then\n # Plain directory present (e.g. first run after adding this import) — migrate\n cp -r .hippo/. /tmp/gh-aw/cache-memory/hippo-store/ 2>/dev/null || true\n rm -rf .hippo\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Migrated existing .hippo/ → cache-memory/hippo-store\"\nelse\n echo \"✅ .hippo already linked to cache-memory/hippo-store\"\nfi\n\n# Initialise if the store has never been set up. Perform a one-time\n# repository scan to seed the memory store with historical incidents.\n# Set HIPPO_SCAN_DAYS to override the default 365-day scan window.\nHIPPO_SCAN_DAYS=\"${HIPPO_SCAN_DAYS:-365}\"\nif [ ! -f \".hippo/config.json\" ]; then\n hippo init --scan --days \"$HIPPO_SCAN_DAYS\"\n echo \"✅ Hippo memory store initialised (scan days: $HIPPO_SCAN_DAYS)\"\nelse\n echo \"✅ Hippo store restored from cache\"\n hippo list 2>/dev/null | head -5 || true\nfi\n\n# One-time bootstrap for repositories with an empty or freshly initialised\n# store. This seeds memory with key project docs and recurring incidents.\nif [ ! -f \".hippo/.gh-aw-bootstrap-v1\" ]; then\n if [ -f \"AGENTS.md\" ]; then\n hippo import --markdown AGENTS.md\n fi\n\n INCIDENTS=(\n 'Recurring incident: Codex auth failures can break agent jobs; verify auth/mode/token setup before reruns.'\n 'Recurring incident: stale workflow .lock.yml files cause churn and CI friction; run make recompile after markdown workflow edits.'\n 'Recurring incident: node: command not found on GPU/self-hosted paths when node runtime/tooling is missing; validate runtimes and PATH early.'\n )\n for incident in \"${INCIDENTS[@]}\"; do\n hippo remember \"$incident\" --tag incident\n done\n\n touch \".hippo/.gh-aw-bootstrap-v1\"\n echo \"✅ Hippo store bootstrap complete (.gh-aw-bootstrap-v1)\"\nfi\n" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index f30bd2d3978..cd6ba77340f 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -566,6 +566,22 @@ jobs: DIFC_PROXY_IMAGE: 'ghcr.io/github/gh-aw-mcpg:v0.3.33' run: | bash "${RUNNER_TEMP}/gh-aw/actions/start_difc_proxy.sh" + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-approved-6f25a3c0-issues-data-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-approved-6f25a3c0-issues-data- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: approved + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" @@ -718,22 +734,6 @@ jobs: /tmp/gh-aw/python/*.py /tmp/gh-aw/python/data/* retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-approved-6f25a3c0-issues-data-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-approved-6f25a3c0-issues-data- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: approved - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index 3f04f8e0727..909e8ee5260 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -554,6 +554,32 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone repo-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: memory/daily-news + TARGET_REPO: ${{ github.repository }} + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries @@ -638,32 +664,6 @@ jobs: name: List downloaded data run: find /tmp/gh-aw/agent/daily-news-data/ -maxdepth 1 -ls - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone repo-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: memory/daily-news - TARGET_REPO: ${{ github.repository }} - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-performance-summary.lock.yml b/.github/workflows/daily-performance-summary.lock.yml index 58efab7342e..b4caf6aa887 100644 --- a/.github/workflows/daily-performance-summary.lock.yml +++ b/.github/workflows/daily-performance-summary.lock.yml @@ -498,19 +498,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: trending-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -527,6 +514,19 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: trending-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-repo-chronicle.lock.yml b/.github/workflows/daily-repo-chronicle.lock.yml index 402a7e378dc..33bba64dbe9 100644 --- a/.github/workflows/daily-repo-chronicle.lock.yml +++ b/.github/workflows/daily-repo-chronicle.lock.yml @@ -489,21 +489,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -520,6 +505,21 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 43be93b638a..1d7503615a0 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -569,11 +569,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download logs from last 24 hours - run: ./gh-aw logs --start-date -1d -o /tmp/gh-aw/aw-mcp/logs - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -590,6 +585,11 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download logs from last 24 hours + run: ./gh-aw logs --start-date -1d -o /tmp/gh-aw/aw-mcp/logs + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 0a3e44ea292..cc1f07f84bf 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -538,6 +538,22 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries @@ -579,22 +595,6 @@ jobs: name: Download integrity-filtered logs run: "mkdir -p /tmp/gh-aw/agent/integrity\nmkdir -p /tmp/gh-aw/cache-memory/security-observability\n\nCACHE_FILE=/tmp/gh-aw/cache-memory/security-observability/filtered-logs.snapshot.json\nRUN_FILE=/tmp/gh-aw/agent/integrity/filtered-logs.json\nFRESH_LOGS=/tmp/gh-aw/agent/integrity/filtered-logs.fresh.json\nEMPTY_DATA='{\"runs\":[],\"summary\":{\"total_runs\":0}}'\nNOW_EPOCH=$(date +%s)\nMAX_CACHE_AGE_SECONDS=$((7 * 24 * 60 * 60))\n\n# Warm start from cached 7-day snapshot when available and fresh.\nif [ -f \"$CACHE_FILE\" ] && jq -e '.runs and .updated_at' \"$CACHE_FILE\" > /dev/null 2>&1; then\n cache_updated_at=$(jq -r '.updated_at' \"$CACHE_FILE\")\n cache_updated_epoch=$(\n date -d \"$cache_updated_at\" +%s 2>/dev/null \\\n || date -j -f \"%Y-%m-%dT%H:%M:%SZ\" \"$cache_updated_at\" +%s 2>/dev/null \\\n || echo 0\n )\n cache_age_seconds=$((NOW_EPOCH - cache_updated_epoch))\n if [ \"$cache_updated_epoch\" -gt 0 ] && [ \"$cache_age_seconds\" -le \"$MAX_CACHE_AGE_SECONDS\" ]; then\n jq '{runs: (.runs // []), summary: (.summary // {\"total_runs\": 0})}' \"$CACHE_FILE\" > \"$RUN_FILE\"\n echo \"✅ Warm cache restored (${cache_age_seconds}s old)\"\n else\n echo \"⚠️ Cache snapshot is stale (${cache_age_seconds}s old); starting fresh\"\n fi\nfi\n\n# Download logs filtered to only runs with DIFC integrity-filtered events.\n# --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking).\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --filtered-integrity --start-date -7d --json -c 200 \\\n --artifacts mcp --timeout 8 \\\n > \"$FRESH_LOGS\" || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' \"$FRESH_LOGS\" > /dev/null 2>&1; then\n echo \"⚠️ No valid logs produced; continuing with empty dataset\"\n echo \"$EMPTY_DATA\" > \"$FRESH_LOGS\"\nfi\n\n# Merge warm-start and fresh runs; fresh entries override warm-cache entries with the same run_id.\nif [ -f \"$RUN_FILE\" ] && jq -e '.runs' \"$RUN_FILE\" > /dev/null 2>&1; then\n jq -s '\n {\n runs: (\n ((.[0].runs // []) + (.[1].runs // []))\n | sort_by(.run_id)\n | group_by(.run_id)\n | map(.[-1])\n ),\n summary: {\n total_runs: (\n ((.[0].runs // []) + (.[1].runs // []) | sort_by(.run_id) | group_by(.run_id) | length)\n )\n }\n }\n ' \"$RUN_FILE\" \"$FRESH_LOGS\" > \"$RUN_FILE.merged\"\n mv \"$RUN_FILE.merged\" \"$RUN_FILE\"\nelse\n mv \"$FRESH_LOGS\" \"$RUN_FILE\"\nfi\n\ncount=$(jq '.runs | length' \"$RUN_FILE\" 2>/dev/null || echo 0)\necho \"✅ Downloaded $count runs with integrity-filtered events\"\n\n# Persist updated 7-day snapshot back to cache-memory every run.\njq -n \\\n --arg updated_at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \\\n --slurpfile payload \"$RUN_FILE\" \\\n '{\n updated_at: $updated_at,\n runs: ($payload[0].runs // []),\n summary: {\n total_runs: (($payload[0].runs // []) | length)\n }\n }' > \"$CACHE_FILE\"" - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-sentrux-report.lock.yml b/.github/workflows/daily-sentrux-report.lock.yml index 358df9ae644..391c59d2ebc 100644 --- a/.github/workflows/daily-sentrux-report.lock.yml +++ b/.github/workflows/daily-sentrux-report.lock.yml @@ -487,14 +487,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install sentrux - run: | - SENTRUX_VERSION="0.5.7" - curl -fsSL -o /tmp/gh-aw/agent/sentrux "https://github.com/sentrux/sentrux/releases/download/v${SENTRUX_VERSION}/sentrux-linux-x86_64" - chmod +x /tmp/gh-aw/agent/sentrux - sudo mv /tmp/gh-aw/agent/sentrux /usr/local/bin/sentrux - sentrux --version - # Repo memory git-based storage configuration from frontmatter processed below - name: Clone repo-memory branch (default) env: @@ -505,6 +497,14 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Install sentrux + run: | + SENTRUX_VERSION="0.5.7" + curl -fsSL -o /tmp/gh-aw/agent/sentrux "https://github.com/sentrux/sentrux/releases/download/v${SENTRUX_VERSION}/sentrux-linux-x86_64" + chmod +x /tmp/gh-aw/agent/sentrux + sudo mv /tmp/gh-aw/agent/sentrux /usr/local/bin/sentrux + sentrux --version + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-spdd-spec-planner.lock.yml b/.github/workflows/daily-spdd-spec-planner.lock.yml index ebab6f4ea5a..0fc01f09269 100644 --- a/.github/workflows/daily-spdd-spec-planner.lock.yml +++ b/.github/workflows/daily-spdd-spec-planner.lock.yml @@ -478,11 +478,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ github.token }} - name: Copy OpenSPDD prompts - run: "set -euo pipefail\n# Resolve the latest OpenSPDD main ref each run via authenticated GitHub API.\n# This intentionally tracks upstream prompt updates while avoiding unauthenticated rate limits.\nOPENSPDD_REF=\"$(gh api repos/gszhangwei/open-spdd/commits/main --jq .sha)\"\ntest -n \"${OPENSPDD_REF}\" || { echo \"::error::Failed to resolve OpenSPDD main ref\"; exit 1; }\nPROMPTS_DIR=\"${GITHUB_WORKSPACE}/.github/copilot-prompts\"\nmkdir -p \"${PROMPTS_DIR}\"\nfor PROMPT in spdd-analysis spdd-reasons-canvas spdd-generate spdd-sync; do\n gh api \\\n -H \"Accept: application/vnd.github.raw\" \\\n \"repos/gszhangwei/open-spdd/contents/.cursor/commands/${PROMPT}.md?ref=${OPENSPDD_REF}\" \\\n > \"${PROMPTS_DIR}/${PROMPT}.md\"\n test -s \"${PROMPTS_DIR}/${PROMPT}.md\" || { echo \"::error::Failed to download ${PROMPT}.md\"; exit 1; }\ndone" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -499,6 +494,11 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - env: + GH_TOKEN: ${{ github.token }} + name: Copy OpenSPDD prompts + run: "set -euo pipefail\n# Resolve the latest OpenSPDD main ref each run via authenticated GitHub API.\n# This intentionally tracks upstream prompt updates while avoiding unauthenticated rate limits.\nOPENSPDD_REF=\"$(gh api repos/gszhangwei/open-spdd/commits/main --jq .sha)\"\ntest -n \"${OPENSPDD_REF}\" || { echo \"::error::Failed to resolve OpenSPDD main ref\"; exit 1; }\nPROMPTS_DIR=\"${GITHUB_WORKSPACE}/.github/copilot-prompts\"\nmkdir -p \"${PROMPTS_DIR}\"\nfor PROMPT in spdd-analysis spdd-reasons-canvas spdd-generate spdd-sync; do\n gh api \\\n -H \"Accept: application/vnd.github.raw\" \\\n \"repos/gszhangwei/open-spdd/contents/.cursor/commands/${PROMPT}.md?ref=${OPENSPDD_REF}\" \\\n > \"${PROMPTS_DIR}/${PROMPT}.md\"\n test -s \"${PROMPTS_DIR}/${PROMPT}.md\" || { echo \"::error::Failed to download ${PROMPT}.md\"; exit 1; }\ndone" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml index f1f8b70ac19..905d9a2f402 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml +++ b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml @@ -564,6 +564,32 @@ jobs: DIFC_PROXY_IMAGE: 'ghcr.io/github/gh-aw-mcpg:v0.3.33' run: | bash "${RUNNER_TEMP}/gh-aw/actions/start_difc_proxy.sh" + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-approved-6f25a3c0-discussions-data-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-approved-6f25a3c0-discussions-data- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: approved + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone repo-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: memory/dataflow-dataset + TARGET_REPO: ${{ github.repository }} + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" @@ -757,32 +783,6 @@ jobs: GITHUB_GRAPHQL_URL: https://localhost:18443/api/graphql GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NODE_EXTRA_CA_CERTS: /tmp/gh-aw/proxy-logs/proxy-tls/ca.crt - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-approved-6f25a3c0-discussions-data-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-approved-6f25a3c0-discussions-data- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: approved - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone repo-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: memory/dataflow-dataset - TARGET_REPO: ${{ github.repository }} - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/dead-code-remover.lock.yml b/.github/workflows/dead-code-remover.lock.yml index f6674a8984e..0c60fa01bcd 100644 --- a/.github/workflows/dead-code-remover.lock.yml +++ b/.github/workflows/dead-code-remover.lock.yml @@ -502,9 +502,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install deadcode analyzer - run: go install golang.org/x/tools/cmd/deadcode@latest - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -521,6 +518,9 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Install deadcode analyzer + run: go install golang.org/x/tools/cmd/deadcode@latest + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index b98c7e12562..0f5a6a91ca8 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -586,6 +586,32 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-discussions-data-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-discussions-data- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone repo-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: memory/deep-report + TARGET_REPO: ${{ github.repository }} + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" @@ -790,32 +816,6 @@ jobs: echo "Weekly issues data available at: /tmp/gh-aw/agent/weekly-issues-data/issues.json" echo "Schema available at: /tmp/gh-aw/agent/weekly-issues-data/issues-schema.json" - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-discussions-data-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-discussions-data- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone repo-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: memory/deep-report - TARGET_REPO: ${{ github.repository }} - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/dependabot-burner.lock.yml b/.github/workflows/dependabot-burner.lock.yml index 27efb954320..479de466423 100644 --- a/.github/workflows/dependabot-burner.lock.yml +++ b/.github/workflows/dependabot-burner.lock.yml @@ -547,6 +547,12 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache configuration from frontmatter processed below + - name: Dependabot burner selection context + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: dependabot-burner-selection-${{ github.run_id }} + path: /tmp/gh-aw/agent/dependabot-burner - env: BURN_OBJECTIVE: ${{ inputs.objective }} name: Prefetch dependabot burner context @@ -554,12 +560,6 @@ jobs: with: script: "const fs = require('fs');\nconst path = require('path');\n\nconst manifestTargets = new Set([\n '.github/workflows/package.json',\n '.github/workflows/package-lock.json',\n '.github/workflows/requirements.txt',\n '.github/workflows/go.mod',\n]);\nconst objective = (process.env.BURN_OBJECTIVE || '').trim() || 'Close grouped Dependabot PRs for generated workflow manifests by updating source workflow markdown and recompiling in one replacement PR.';\nconst outPath = '/tmp/gh-aw/agent/dependabot-burner/context.json';\n\nfunction parseBumpTitle(title) {\n const match = String(title || '').match(/^Bump\\s+(.+?)\\s+from\\s+([^\\s]+)\\s+to\\s+([^\\s]+)$/i);\n if (!match) {\n return {\n dependency_name: String(title || '').trim(),\n current_version: '',\n target_version: '',\n title_parse_mode: 'fallback',\n };\n }\n return {\n dependency_name: match[1],\n current_version: match[2],\n target_version: match[3],\n title_parse_mode: 'parsed',\n };\n}\n\nfunction normalizeManifestFamily(filename) {\n if (filename.includes('package')) {\n return 'npm';\n }\n if (filename.endsWith('requirements.txt')) {\n return 'pip';\n }\n if (filename.endsWith('go.mod')) {\n return 'go';\n }\n return 'other';\n}\n\nfunction summarizeFamilies(files) {\n return [...new Set((files || []).map(normalizeManifestFamily))].sort();\n}\n\nfunction getTriggerPRNumber() {\n if (context.payload.pull_request?.number) {\n return Number(context.payload.pull_request.number);\n }\n if (context.payload.issue?.pull_request && context.payload.issue?.number) {\n return Number(context.payload.issue.number);\n }\n return null;\n}\n\nasync function loadPullFiles(pullNumber) {\n const files = await github.paginate(github.rest.pulls.listFiles, {\n owner: context.repo.owner,\n repo: context.repo.repo,\n pull_number: pullNumber,\n per_page: 100,\n });\n return files.map((file) => file.filename).filter((filename) => manifestTargets.has(filename));\n}\n\nasync function listOpenDependabotPRs() {\n const pulls = await github.paginate(github.rest.pulls.list, {\n owner: context.repo.owner,\n repo: context.repo.repo,\n state: 'open',\n per_page: 100,\n });\n\n const candidates = [];\n for (const pull of pulls) {\n const author = pull.user?.login || '';\n if (author !== 'dependabot[bot]' && author !== 'app/dependabot') {\n continue;\n }\n\n const manifestFiles = await loadPullFiles(pull.number);\n if (manifestFiles.length === 0) {\n continue;\n }\n\n const parsed = parseBumpTitle(pull.title);\n candidates.push({\n number: pull.number,\n title: pull.title,\n dependency_name: parsed.dependency_name,\n current_version: parsed.current_version,\n target_version: parsed.target_version,\n title_parse_mode: parsed.title_parse_mode,\n manifest_files: manifestFiles,\n manifest_families: summarizeFamilies(manifestFiles),\n created_at: pull.created_at,\n updated_at: pull.updated_at,\n url: pull.html_url,\n });\n }\n\n return candidates.sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime());\n}\n\nasync function listRecentClosedBurnerPRs() {\n const pulls = await github.paginate(github.rest.pulls.list, {\n owner: context.repo.owner,\n repo: context.repo.repo,\n state: 'closed',\n per_page: 100,\n });\n\n return pulls\n .filter((pull) => pull.title?.startsWith('[dependabot-burner] ') && !pull.merged_at)\n .slice(0, 20)\n .map((pull) => ({\n number: pull.number,\n title: pull.title,\n body: pull.body || '',\n url: pull.html_url,\n closed_at: pull.closed_at,\n created_at: pull.created_at,\n }))\n .sort((a, b) => new Date(b.closed_at || b.created_at).getTime() - new Date(a.closed_at || a.created_at).getTime());\n}\n\nconst triggerPRNumber = getTriggerPRNumber();\nconst openPRs = await listOpenDependabotPRs();\nconst recentFailedBurns = await listRecentClosedBurnerPRs();\n\nlet triggerPR = openPRs.find((pull) => pull.number === triggerPRNumber) || null;\nlet selectionReason = triggerPRNumber ? 'slash-command-trigger-not-in-scope' : 'bundle-all-open-manifest-prs';\nlet selectedPRs = openPRs;\n\nif (triggerPRNumber) {\n if (!triggerPR) {\n const manifestFiles = await loadPullFiles(triggerPRNumber);\n if (manifestFiles.length > 0) {\n const pull = await github.rest.pulls.get({\n owner: context.repo.owner,\n repo: context.repo.repo,\n pull_number: triggerPRNumber,\n });\n const parsed = parseBumpTitle(pull.data.title);\n triggerPR = {\n number: pull.data.number,\n title: pull.data.title,\n dependency_name: parsed.dependency_name,\n current_version: parsed.current_version,\n target_version: parsed.target_version,\n title_parse_mode: parsed.title_parse_mode,\n manifest_files: manifestFiles,\n manifest_families: summarizeFamilies(manifestFiles),\n created_at: pull.data.created_at,\n updated_at: pull.data.updated_at,\n url: pull.data.html_url,\n };\n }\n }\n\n if (triggerPR) {\n const triggerFiles = new Set(triggerPR.manifest_files || []);\n selectedPRs = openPRs.filter((pull) => {\n if (pull.number === triggerPR.number) {\n return true;\n }\n return (pull.manifest_files || []).some((file) => triggerFiles.has(file));\n });\n selectionReason = 'slash-command-similar-prs';\n } else {\n selectedPRs = [];\n }\n}\n\nconst payload = {\n objective,\n trigger_event: context.eventName,\n trigger_pr_number: triggerPRNumber,\n trigger_pr: triggerPR,\n selection_reason: selectionReason,\n open_pr_count: openPRs.length,\n selected_batch_pr_numbers: selectedPRs.map((pull) => pull.number),\n selected_batch_dependencies: selectedPRs.map((pull) => ({\n pr_number: pull.number,\n dependency_name: pull.dependency_name,\n current_version: pull.current_version,\n target_version: pull.target_version,\n title_parse_mode: pull.title_parse_mode,\n manifest_files: pull.manifest_files,\n manifest_families: pull.manifest_families,\n title: pull.title,\n url: pull.url,\n })),\n related_prs: triggerPRNumber ? selectedPRs.filter((pull) => pull.number !== triggerPRNumber) : [],\n recent_failed_burns: recentFailedBurns,\n};\n\nfs.mkdirSync(path.dirname(outPath), { recursive: true });\nfs.writeFileSync(outPath, JSON.stringify(payload, null, 2) + '\\n', 'utf8');\nconsole.log(JSON.stringify(payload, null, 2));" - # Cache configuration from frontmatter processed below - - name: Dependabot burner selection context - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: dependabot-burner-selection-${{ github.run_id }} - path: /tmp/gh-aw/agent/dependabot-burner - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/detection-analysis-report.lock.yml b/.github/workflows/detection-analysis-report.lock.yml index 63d6e8f1d8c..12fa1379d0d 100644 --- a/.github/workflows/detection-analysis-report.lock.yml +++ b/.github/workflows/detection-analysis-report.lock.yml @@ -527,19 +527,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: trending-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -556,6 +543,19 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n# Reinstall chart libraries every run so chart generation never depends on stale state.\n/tmp/gh-aw/agent/venv/bin/pip install --quiet --upgrade --force-reinstall numpy pandas matplotlib seaborn scipy\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy,pandas,matplotlib,seaborn,scipy;print('chart-libraries-ready')\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: trending-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/github-mcp-structural-analysis.lock.yml b/.github/workflows/github-mcp-structural-analysis.lock.yml index 58148480f45..14a48ba9230 100644 --- a/.github/workflows/github-mcp-structural-analysis.lock.yml +++ b/.github/workflows/github-mcp-structural-analysis.lock.yml @@ -495,21 +495,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -526,6 +511,21 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index f95ed429765..17dcbee99b1 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -532,9 +532,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Fetch recent changes - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n\n# Determine scan scope: Monday = full weekly scan, other weekdays = daily\nDAY=$(date +%u)\nif [ \"$DAY\" -eq 1 ]; then\n SINCE=\"7 days ago\"\n SCOPE=\"weekly\"\nelse\n SINCE=\"24 hours ago\"\n SCOPE=\"daily\"\nfi\n\necho \"Scan scope: $SCOPE (since: $SINCE)\"\n\n# Fetch recent commits (all files) — includes file names for context\ngit log --since=\"$SINCE\" --oneline --name-only \\\n > /tmp/gh-aw/agent/recent-commits.txt\n\n# Fetch commits that touched docs\ngit log --since=\"$SINCE\" --name-only \\\n --format=\"%H %s\" -- 'docs/**/*.md' 'docs/**/*.mdx' \\\n > /tmp/gh-aw/agent/doc-changes.txt\n\necho \"Recent commits: $(wc -l < /tmp/gh-aw/agent/recent-commits.txt)\"\necho \"Doc file changes: $(wc -l < /tmp/gh-aw/agent/doc-changes.txt)\"\necho \"$SCOPE\" > /tmp/gh-aw/agent/scan-scope.txt" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -561,6 +558,9 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: false run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Fetch recent changes + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n\n# Determine scan scope: Monday = full weekly scan, other weekdays = daily\nDAY=$(date +%u)\nif [ \"$DAY\" -eq 1 ]; then\n SINCE=\"7 days ago\"\n SCOPE=\"weekly\"\nelse\n SINCE=\"24 hours ago\"\n SCOPE=\"daily\"\nfi\n\necho \"Scan scope: $SCOPE (since: $SINCE)\"\n\n# Fetch recent commits (all files) — includes file names for context\ngit log --since=\"$SINCE\" --oneline --name-only \\\n > /tmp/gh-aw/agent/recent-commits.txt\n\n# Fetch commits that touched docs\ngit log --since=\"$SINCE\" --name-only \\\n --format=\"%H %s\" -- 'docs/**/*.md' 'docs/**/*.mdx' \\\n > /tmp/gh-aw/agent/doc-changes.txt\n\necho \"Recent commits: $(wc -l < /tmp/gh-aw/agent/recent-commits.txt)\"\necho \"Doc file changes: $(wc -l < /tmp/gh-aw/agent/doc-changes.txt)\"\necho \"$SCOPE\" > /tmp/gh-aw/agent/scan-scope.txt" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/go-logger.lock.yml b/.github/workflows/go-logger.lock.yml index 7ac373689ae..11580ed2142 100644 --- a/.github/workflows/go-logger.lock.yml +++ b/.github/workflows/go-logger.lock.yml @@ -490,18 +490,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - id: logger_manifest - name: Build deterministic logger manifest - run: "set -euo pipefail\ncache_dir=\"/tmp/gh-aw/cache-memory/go-logger\"\nout_dir=\"/tmp/gh-aw/agent/go-logger\"\nmkdir -p \"$cache_dir\" \"$out_dir\"\n\ncurrent_sha=\"$(git rev-parse HEAD)\"\ncurrent_files=\"$out_dir/current-files.txt\"\nprocessed_files=\"$cache_dir/processed-files.txt\"\nfind pkg -name '*.go' -type f ! -name '*_test.go' | sort > \"$current_files\"\n[ -f \"$processed_files\" ] || : > \"$processed_files\"\n\nnew_files=\"$out_dir/new-files.txt\"\ncomm -23 \"$current_files\" \"$processed_files\" > \"$new_files\" || true\n\nlast_sha=\"\"\nif [ -f \"$cache_dir/last-run.json\" ]; then\n last_sha=\"$(jq -r '.commit_sha // empty' \"$cache_dir/last-run.json\" 2>/dev/null || true)\"\nfi\n\nfiles_needing_logger=\"$out_dir/files-needing-logger.txt\"\nfiles_missing_logger_import=\"$out_dir/files-missing-logger-import.txt\"\ncall_sites=\"$out_dir/call-sites.tsv\"\n: > \"$files_needing_logger\"\n: > \"$files_missing_logger_import\"\n: > \"$call_sites\"\n\nwhile IFS= read -r rel; do\n [ -f \"$rel\" ] || continue\n if ! grep -q \"var log = logger.New\" \"$rel\"; then\n echo \"$rel\" >> \"$files_needing_logger\"\n fi\n if grep -q \"log\\\\.\" \"$rel\" && ! grep -q '\"github.com/github/gh-aw/pkg/logger\"' \"$rel\"; then\n echo \"$rel\" >> \"$files_missing_logger_import\"\n fi\n while IFS=: read -r line_number match_line; do\n function_name=\"$(printf '%s' \"$match_line\" | sed -E 's/^[[:space:]]*func[[:space:]]+([A-Za-z0-9_]+).*/\\\\1/')\"\n printf \"%s\\\\t%s\\\\t%s\\\\n\" \"$rel\" \"$line_number\" \"$function_name\" >> \"$call_sites\"\n done < <(grep -nE \"^[[:space:]]*func[[:space:]]+[A-Za-z0-9_]+\" \"$rel\" || true)\ndone < \"$current_files\"\n\n# Write each JSON payload to a file to avoid exceeding ARG_MAX with large datasets.\njq -R -s 'split(\"\\n\") | map(select(length > 0))' \"$files_needing_logger\" \\\n > \"$out_dir/files-needing-logger.json\"\njq -R -s 'split(\"\\n\") | map(select(length > 0))' \"$files_missing_logger_import\" \\\n > \"$out_dir/files-missing-logger-import.json\"\njq -R -s 'split(\"\\n\") | map(select(length > 0) | split(\"\\t\") | {file: .[0], line: (.[1] | tonumber), function: .[2]})' \"$call_sites\" \\\n > \"$out_dir/candidate-call-sites.json\"\n\n# Build manifest by reading files via --slurpfile (no large args on argv).\njq -n \\\n --slurpfile files_needing_logger \"$out_dir/files-needing-logger.json\" \\\n --slurpfile missing_logger_import \"$out_dir/files-missing-logger-import.json\" \\\n --slurpfile candidate_call_sites \"$out_dir/candidate-call-sites.json\" \\\n '{files_needing_logger: $files_needing_logger[0], missing_logger_import: $missing_logger_import[0], candidate_call_sites: $candidate_call_sites[0]}' \\\n > \"$out_dir/manifest.json\"\n\nshould_run=true\nif [ \"$current_sha\" = \"$last_sha\" ] && [ ! -s \"$new_files\" ]; then\n should_run=false\nfi\necho \"{\\\"should_run\\\": \\\"$should_run\\\", \\\"current_sha\\\": \\\"$current_sha\\\", \\\"last_sha\\\": \\\"$last_sha\\\", \\\"manifest\\\": \\\"$out_dir/manifest.json\\\", \\\"new_files\\\": \\\"$new_files\\\"}\" > \"$out_dir/preflight.json\"\necho \"should_run=$should_run\" >> \"$GITHUB_OUTPUT\"\n" - - name: Setup Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 - with: - cache: true - go-version-file: go.mod - - name: Install npm dependencies - run: npm ci - working-directory: ./actions/setup/js - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -518,6 +506,18 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - id: logger_manifest + name: Build deterministic logger manifest + run: "set -euo pipefail\ncache_dir=\"/tmp/gh-aw/cache-memory/go-logger\"\nout_dir=\"/tmp/gh-aw/agent/go-logger\"\nmkdir -p \"$cache_dir\" \"$out_dir\"\n\ncurrent_sha=\"$(git rev-parse HEAD)\"\ncurrent_files=\"$out_dir/current-files.txt\"\nprocessed_files=\"$cache_dir/processed-files.txt\"\nfind pkg -name '*.go' -type f ! -name '*_test.go' | sort > \"$current_files\"\n[ -f \"$processed_files\" ] || : > \"$processed_files\"\n\nnew_files=\"$out_dir/new-files.txt\"\ncomm -23 \"$current_files\" \"$processed_files\" > \"$new_files\" || true\n\nlast_sha=\"\"\nif [ -f \"$cache_dir/last-run.json\" ]; then\n last_sha=\"$(jq -r '.commit_sha // empty' \"$cache_dir/last-run.json\" 2>/dev/null || true)\"\nfi\n\nfiles_needing_logger=\"$out_dir/files-needing-logger.txt\"\nfiles_missing_logger_import=\"$out_dir/files-missing-logger-import.txt\"\ncall_sites=\"$out_dir/call-sites.tsv\"\n: > \"$files_needing_logger\"\n: > \"$files_missing_logger_import\"\n: > \"$call_sites\"\n\nwhile IFS= read -r rel; do\n [ -f \"$rel\" ] || continue\n if ! grep -q \"var log = logger.New\" \"$rel\"; then\n echo \"$rel\" >> \"$files_needing_logger\"\n fi\n if grep -q \"log\\\\.\" \"$rel\" && ! grep -q '\"github.com/github/gh-aw/pkg/logger\"' \"$rel\"; then\n echo \"$rel\" >> \"$files_missing_logger_import\"\n fi\n while IFS=: read -r line_number match_line; do\n function_name=\"$(printf '%s' \"$match_line\" | sed -E 's/^[[:space:]]*func[[:space:]]+([A-Za-z0-9_]+).*/\\\\1/')\"\n printf \"%s\\\\t%s\\\\t%s\\\\n\" \"$rel\" \"$line_number\" \"$function_name\" >> \"$call_sites\"\n done < <(grep -nE \"^[[:space:]]*func[[:space:]]+[A-Za-z0-9_]+\" \"$rel\" || true)\ndone < \"$current_files\"\n\n# Write each JSON payload to a file to avoid exceeding ARG_MAX with large datasets.\njq -R -s 'split(\"\\n\") | map(select(length > 0))' \"$files_needing_logger\" \\\n > \"$out_dir/files-needing-logger.json\"\njq -R -s 'split(\"\\n\") | map(select(length > 0))' \"$files_missing_logger_import\" \\\n > \"$out_dir/files-missing-logger-import.json\"\njq -R -s 'split(\"\\n\") | map(select(length > 0) | split(\"\\t\") | {file: .[0], line: (.[1] | tonumber), function: .[2]})' \"$call_sites\" \\\n > \"$out_dir/candidate-call-sites.json\"\n\n# Build manifest by reading files via --slurpfile (no large args on argv).\njq -n \\\n --slurpfile files_needing_logger \"$out_dir/files-needing-logger.json\" \\\n --slurpfile missing_logger_import \"$out_dir/files-missing-logger-import.json\" \\\n --slurpfile candidate_call_sites \"$out_dir/candidate-call-sites.json\" \\\n '{files_needing_logger: $files_needing_logger[0], missing_logger_import: $missing_logger_import[0], candidate_call_sites: $candidate_call_sites[0]}' \\\n > \"$out_dir/manifest.json\"\n\nshould_run=true\nif [ \"$current_sha\" = \"$last_sha\" ] && [ ! -s \"$new_files\" ]; then\n should_run=false\nfi\necho \"{\\\"should_run\\\": \\\"$should_run\\\", \\\"current_sha\\\": \\\"$current_sha\\\", \\\"last_sha\\\": \\\"$last_sha\\\", \\\"manifest\\\": \\\"$out_dir/manifest.json\\\", \\\"new_files\\\": \\\"$new_files\\\"}\" > \"$out_dir/preflight.json\"\necho \"should_run=$should_run\" >> \"$GITHUB_OUTPUT\"\n" + - name: Setup Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + cache: true + go-version-file: go.mod + - name: Install npm dependencies + run: npm ci + working-directory: ./actions/setup/js + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index 7d9dc7a0ad0..4706ab8833c 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -524,12 +524,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download SBOM from GitHub Dependency Graph API - run: "set -e\necho \"📦 Downloading SBOM from GitHub Dependency Graph API...\"\n\n# Download SBOM using gh CLI (requires contents: read permission)\ngh api \\\n -H \"Accept: application/vnd.github+json\" \\\n -H \"X-GitHub-Api-Version: 2022-11-28\" \\\n \"/repos/$GITHUB_REPOSITORY/dependency-graph/sbom\" \\\n > /tmp/gh-aw/agent/sbom.json\n\necho \"✅ SBOM downloaded successfully to /tmp/gh-aw/agent/sbom.json\"\n\n# Show SBOM summary\nif command -v jq &> /dev/null; then\n PACKAGE_COUNT=$(jq '.sbom.packages | length' /tmp/gh-aw/agent/sbom.json 2>/dev/null || echo \"unknown\")\n echo \"📊 SBOM contains ${PACKAGE_COUNT} packages\"\nfi" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -546,6 +540,12 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download SBOM from GitHub Dependency Graph API + run: "set -e\necho \"📦 Downloading SBOM from GitHub Dependency Graph API...\"\n\n# Download SBOM using gh CLI (requires contents: read permission)\ngh api \\\n -H \"Accept: application/vnd.github+json\" \\\n -H \"X-GitHub-Api-Version: 2022-11-28\" \\\n \"/repos/$GITHUB_REPOSITORY/dependency-graph/sbom\" \\\n > /tmp/gh-aw/agent/sbom.json\n\necho \"✅ SBOM downloaded successfully to /tmp/gh-aw/agent/sbom.json\"\n\n# Show SBOM summary\nif command -v jq &> /dev/null; then\n PACKAGE_COUNT=$(jq '.sbom.packages | length' /tmp/gh-aw/agent/sbom.json 2>/dev/null || echo \"unknown\")\n echo \"📊 SBOM contains ${PACKAGE_COUNT} packages\"\nfi" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/hippo-embed.lock.yml b/.github/workflows/hippo-embed.lock.yml index beeb2f155b1..948ea28d2d2 100644 --- a/.github/workflows/hippo-embed.lock.yml +++ b/.github/workflows/hippo-embed.lock.yml @@ -477,14 +477,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install hippo-memory - run: | - npm install -g hippo-memory - - name: Initialize hippo store - run: "# Symlink .hippo into cache-memory so the SQLite store persists across runs.\n# All writes to .hippo/ land in /tmp/gh-aw/cache-memory/hippo-store/ and are\n# saved/restored automatically by the cache-memory mechanism.\nmkdir -p /tmp/gh-aw/cache-memory/hippo-store\n\nif [ ! -e \".hippo\" ]; then\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Created .hippo → cache-memory/hippo-store\"\nelif [ -d \".hippo\" ] && [ ! -L \".hippo\" ]; then\n # Plain directory present (e.g. first run after adding this import) — migrate\n cp -r .hippo/. /tmp/gh-aw/cache-memory/hippo-store/ 2>/dev/null || true\n rm -rf .hippo\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Migrated existing .hippo/ → cache-memory/hippo-store\"\nelse\n echo \"✅ .hippo already linked to cache-memory/hippo-store\"\nfi\n\n# Initialise if the store has never been set up. Perform a one-time\n# repository scan to seed the memory store with historical incidents.\n# Set HIPPO_SCAN_DAYS to override the default 365-day scan window.\nHIPPO_SCAN_DAYS=\"${HIPPO_SCAN_DAYS:-365}\"\nif [ ! -f \".hippo/config.json\" ]; then\n hippo init --scan --days \"$HIPPO_SCAN_DAYS\"\n echo \"✅ Hippo memory store initialised (scan days: $HIPPO_SCAN_DAYS)\"\nelse\n echo \"✅ Hippo store restored from cache\"\n hippo list 2>/dev/null | head -5 || true\nfi\n\n# One-time bootstrap for repositories with an empty or freshly initialised\n# store. This seeds memory with key project docs and recurring incidents.\nif [ ! -f \".hippo/.gh-aw-bootstrap-v1\" ]; then\n if [ -f \"AGENTS.md\" ]; then\n hippo import --markdown AGENTS.md\n fi\n\n INCIDENTS=(\n 'Recurring incident: Codex auth failures can break agent jobs; verify auth/mode/token setup before reruns.'\n 'Recurring incident: stale workflow .lock.yml files cause churn and CI friction; run make recompile after markdown workflow edits.'\n 'Recurring incident: node: command not found on GPU/self-hosted paths when node runtime/tooling is missing; validate runtimes and PATH early.'\n )\n for incident in \"${INCIDENTS[@]}\"; do\n hippo remember \"$incident\" --tag incident\n done\n\n touch \".hippo/.gh-aw-bootstrap-v1\"\n echo \"✅ Hippo store bootstrap complete (.gh-aw-bootstrap-v1)\"\nfi\n" - - name: Install @xenova/transformers - run: npm install -g @xenova/transformers - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -501,6 +493,14 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Install hippo-memory + run: | + npm install -g hippo-memory + - name: Initialize hippo store + run: "# Symlink .hippo into cache-memory so the SQLite store persists across runs.\n# All writes to .hippo/ land in /tmp/gh-aw/cache-memory/hippo-store/ and are\n# saved/restored automatically by the cache-memory mechanism.\nmkdir -p /tmp/gh-aw/cache-memory/hippo-store\n\nif [ ! -e \".hippo\" ]; then\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Created .hippo → cache-memory/hippo-store\"\nelif [ -d \".hippo\" ] && [ ! -L \".hippo\" ]; then\n # Plain directory present (e.g. first run after adding this import) — migrate\n cp -r .hippo/. /tmp/gh-aw/cache-memory/hippo-store/ 2>/dev/null || true\n rm -rf .hippo\n ln -s /tmp/gh-aw/cache-memory/hippo-store .hippo\n echo \"🔗 Migrated existing .hippo/ → cache-memory/hippo-store\"\nelse\n echo \"✅ .hippo already linked to cache-memory/hippo-store\"\nfi\n\n# Initialise if the store has never been set up. Perform a one-time\n# repository scan to seed the memory store with historical incidents.\n# Set HIPPO_SCAN_DAYS to override the default 365-day scan window.\nHIPPO_SCAN_DAYS=\"${HIPPO_SCAN_DAYS:-365}\"\nif [ ! -f \".hippo/config.json\" ]; then\n hippo init --scan --days \"$HIPPO_SCAN_DAYS\"\n echo \"✅ Hippo memory store initialised (scan days: $HIPPO_SCAN_DAYS)\"\nelse\n echo \"✅ Hippo store restored from cache\"\n hippo list 2>/dev/null | head -5 || true\nfi\n\n# One-time bootstrap for repositories with an empty or freshly initialised\n# store. This seeds memory with key project docs and recurring incidents.\nif [ ! -f \".hippo/.gh-aw-bootstrap-v1\" ]; then\n if [ -f \"AGENTS.md\" ]; then\n hippo import --markdown AGENTS.md\n fi\n\n INCIDENTS=(\n 'Recurring incident: Codex auth failures can break agent jobs; verify auth/mode/token setup before reruns.'\n 'Recurring incident: stale workflow .lock.yml files cause churn and CI friction; run make recompile after markdown workflow edits.'\n 'Recurring incident: node: command not found on GPU/self-hosted paths when node runtime/tooling is missing; validate runtimes and PATH early.'\n )\n for incident in \"${INCIDENTS[@]}\"; do\n hippo remember \"$incident\" --tag incident\n done\n\n touch \".hippo/.gh-aw-bootstrap-v1\"\n echo \"✅ Hippo store bootstrap complete (.gh-aw-bootstrap-v1)\"\nfi\n" + - name: Install @xenova/transformers + run: npm install -g @xenova/transformers + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/jsweep.lock.yml b/.github/workflows/jsweep.lock.yml index ffe7845fc88..c3b4df85eab 100644 --- a/.github/workflows/jsweep.lock.yml +++ b/.github/workflows/jsweep.lock.yml @@ -488,10 +488,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install Node.js dependencies - run: npm install - working-directory: actions/setup/js - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -508,6 +504,10 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Install Node.js dependencies + run: npm install + working-directory: actions/setup/js + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/org-health-report.lock.yml b/.github/workflows/org-health-report.lock.yml index 1eacb217610..134f26f4796 100644 --- a/.github/workflows/org-health-report.lock.yml +++ b/.github/workflows/org-health-report.lock.yml @@ -494,21 +494,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -525,6 +510,21 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: approved run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 420d5b1df3e..bc26c1691f1 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -485,21 +485,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -516,6 +501,21 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 3af0ea8ea48..d4f7e62e46e 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -537,6 +537,29 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache configuration from frontmatter processed below + - name: Save prompt clustering data to cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: prompt-clustering-cache-${{ github.run_id }} + path: /tmp/gh-aw/agent/prompt-cache + restore-keys: prompt-clustering-cache- + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" @@ -570,29 +593,6 @@ jobs: name: Download workflow logs for PR analysis run: "# Create logs directory\nmkdir -p /tmp/gh-aw/agent/workflow-logs\n\necho \"Downloading workflow logs to extract turn counts...\"\n\n# Download logs for the last 30 days of copilot workflows\n# This will give us the aw_info.json which contains turn counts\n./gh-aw logs --engine copilot --start-date -30d -o /tmp/gh-aw/agent/workflow-logs\n\n# Verify logs were downloaded\necho \"Downloaded workflow logs:\"\nfind /tmp/gh-aw/agent/workflow-logs -maxdepth 1 -ls" - # Cache configuration from frontmatter processed below - - name: Save prompt clustering data to cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: prompt-clustering-cache-${{ github.run_id }} - path: /tmp/gh-aw/agent/prompt-cache - restore-keys: prompt-clustering-cache- - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/python-data-charts.lock.yml b/.github/workflows/python-data-charts.lock.yml index 13fb08564b4..27b713f6292 100644 --- a/.github/workflows/python-data-charts.lock.yml +++ b/.github/workflows/python-data-charts.lock.yml @@ -520,21 +520,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -551,6 +536,21 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/repository-quality-improver.lock.yml b/.github/workflows/repository-quality-improver.lock.yml index 4cc9888f996..ddd8225acad 100644 --- a/.github/workflows/repository-quality-improver.lock.yml +++ b/.github/workflows/repository-quality-improver.lock.yml @@ -491,9 +491,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Collect quality metrics - run: "mkdir -p /tmp/gh-aw/agent\n{\n echo \"## Focus Area History\"\n if [ -f /tmp/gh-aw/cache-memory-focus-areas/history.json ]; then\n cat /tmp/gh-aw/cache-memory-focus-areas/history.json\n else\n echo '{\"runs\":[],\"recent_areas\":[],\"statistics\":{\"total_runs\":0,\"custom_rate\":0,\"reuse_rate\":0,\"unique_areas_explored\":0}}'\n fi\n\n echo \"\"\n echo \"## Code Metrics\"\n echo \"### Largest Go source files (top 20)\"\n find . -type f -name \"*.go\" ! -name \"*_test.go\" ! -path \"./.git/*\" | xargs wc -l 2>/dev/null | sort -rn | head -21 | tail -20\n\n echo \"### Test ratio\"\n TEST_LOC=$(find . -type f -name \"*_test.go\" ! -path \"./.git/*\" | xargs wc -l 2>/dev/null | tail -1 | awk '{print $1}')\n SRC_LOC=$(find . -type f -name \"*.go\" ! -name \"*_test.go\" ! -path \"./.git/*\" | xargs wc -l 2>/dev/null | tail -1 | awk '{print $1}')\n echo \"Test LOC: $TEST_LOC | Source LOC: $SRC_LOC\"\n\n echo \"### Directory file counts\"\n for dir in cmd pkg docs .github; do\n if [ -d \"$dir\" ]; then\n echo \"$dir: $(find \"$dir\" -type f | wc -l) files\"\n fi\n done\n\n echo \"### TODO/FIXME count\"\n grep -r \"TODO\\|FIXME\" --include=\"*.go\" --include=\"*.cjs\" . 2>/dev/null | wc -l\n\n echo \"### README size\"\n wc -l README.md 2>/dev/null || echo \"No README.md\"\n} > /tmp/gh-aw/agent/analysis-context.md\necho \"✅ Quality metrics collected → /tmp/gh-aw/agent/analysis-context.md\"" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory (focus-areas) run: | @@ -511,6 +508,9 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory-focus-areas GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Collect quality metrics + run: "mkdir -p /tmp/gh-aw/agent\n{\n echo \"## Focus Area History\"\n if [ -f /tmp/gh-aw/cache-memory-focus-areas/history.json ]; then\n cat /tmp/gh-aw/cache-memory-focus-areas/history.json\n else\n echo '{\"runs\":[],\"recent_areas\":[],\"statistics\":{\"total_runs\":0,\"custom_rate\":0,\"reuse_rate\":0,\"unique_areas_explored\":0}}'\n fi\n\n echo \"\"\n echo \"## Code Metrics\"\n echo \"### Largest Go source files (top 20)\"\n find . -type f -name \"*.go\" ! -name \"*_test.go\" ! -path \"./.git/*\" | xargs wc -l 2>/dev/null | sort -rn | head -21 | tail -20\n\n echo \"### Test ratio\"\n TEST_LOC=$(find . -type f -name \"*_test.go\" ! -path \"./.git/*\" | xargs wc -l 2>/dev/null | tail -1 | awk '{print $1}')\n SRC_LOC=$(find . -type f -name \"*.go\" ! -name \"*_test.go\" ! -path \"./.git/*\" | xargs wc -l 2>/dev/null | tail -1 | awk '{print $1}')\n echo \"Test LOC: $TEST_LOC | Source LOC: $SRC_LOC\"\n\n echo \"### Directory file counts\"\n for dir in cmd pkg docs .github; do\n if [ -d \"$dir\" ]; then\n echo \"$dir: $(find \"$dir\" -type f | wc -l) files\"\n fi\n done\n\n echo \"### TODO/FIXME count\"\n grep -r \"TODO\\|FIXME\" --include=\"*.go\" --include=\"*.cjs\" . 2>/dev/null | wc -l\n\n echo \"### README size\"\n wc -l README.md 2>/dev/null || echo \"No README.md\"\n} > /tmp/gh-aw/agent/analysis-context.md\necho \"✅ Quality metrics collected → /tmp/gh-aw/agent/analysis-context.md\"" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index 85a681769ea..4e631c992bc 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -520,11 +520,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download logs from last 24 hours - run: ./gh-aw logs --start-date -1d -o /tmp/gh-aw/aw-mcp/logs - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -541,6 +536,11 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download logs from last 24 hours + run: ./gh-aw logs --start-date -1d -o /tmp/gh-aw/aw-mcp/logs + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/slide-deck-maintainer.lock.yml b/.github/workflows/slide-deck-maintainer.lock.yml index 8e46f31fd68..ecb319003a3 100644 --- a/.github/workflows/slide-deck-maintainer.lock.yml +++ b/.github/workflows/slide-deck-maintainer.lock.yml @@ -509,17 +509,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - cache: npm - cache-dependency-path: docs/package-lock.json - node-version: "24" - - name: Install Marp dependencies - run: |- - cd docs - npm ci - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -536,6 +525,17 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + cache: npm + cache-dependency-path: docs/package-lock.json + node-version: "24" + - name: Install Marp dependencies + run: |- + cd docs + npm ci + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/stale-repo-identifier.lock.yml b/.github/workflows/stale-repo-identifier.lock.yml index dd4f77c4ae7..be946a6b6f5 100644 --- a/.github/workflows/stale-repo-identifier.lock.yml +++ b/.github/workflows/stale-repo-identifier.lock.yml @@ -526,6 +526,22 @@ jobs: DIFC_PROXY_IMAGE: 'ghcr.io/github/gh-aw-mcpg:v0.3.33' run: | bash "${RUNNER_TEMP}/gh-aw/actions/start_difc_proxy.sh" + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-approved-6f25a3c0-stale-repos-analysis-${{ github.workflow }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-approved-6f25a3c0-stale-repos-analysis-${{ github.workflow }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: approved + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Setup Python environment run: | mkdir -p /tmp/gh-aw/python/{data,charts,artifacts} @@ -586,22 +602,6 @@ jobs: GITHUB_GRAPHQL_URL: https://localhost:18443/api/graphql INACTIVE_REPOS: ${{ steps.stale-repos.outputs.inactiveRepos }} NODE_EXTRA_CA_CERTS: /tmp/gh-aw/proxy-logs/proxy-tls/ca.crt - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-approved-6f25a3c0-stale-repos-analysis-${{ github.workflow }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-approved-6f25a3c0-stale-repos-analysis-${{ github.workflow }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: approved - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/static-analysis-report.lock.yml b/.github/workflows/static-analysis-report.lock.yml index a747f46eeff..7a0d0134c85 100644 --- a/.github/workflows/static-analysis-report.lock.yml +++ b/.github/workflows/static-analysis-report.lock.yml @@ -530,18 +530,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Build gh-aw from source - run: | - set -e - make build - "$GITHUB_WORKSPACE/gh-aw" --version - - name: Pull static analysis Docker images - run: "set -e\necho \"Pulling Docker images for static analysis tools...\"\n\n# Pull zizmor Docker image\necho \"Pulling zizmor image...\"\ndocker pull ghcr.io/zizmorcore/zizmor:latest\n\n# Pull poutine Docker image\necho \"Pulling poutine image...\"\ndocker pull ghcr.io/boostsecurityio/poutine:latest\n\n# Pull runner-guard Docker image\necho \"Pulling runner-guard image...\"\ndocker pull ghcr.io/vigilant-llc/runner-guard:latest\n\necho \"All static analysis Docker images pulled successfully\"\n" - - name: Verify static analysis tools - run: "set -e\necho \"Verifying static analysis tools are available...\"\n\n# Verify zizmor\necho \"Testing zizmor...\"\ndocker run --rm ghcr.io/zizmorcore/zizmor:latest --version || echo \"Warning: zizmor version check failed\"\n\n# Verify poutine\necho \"Testing poutine...\"\ndocker run --rm ghcr.io/boostsecurityio/poutine:latest --version || echo \"Warning: poutine version check failed\"\n\n# Verify runner-guard\necho \"Testing runner-guard...\"\ndocker run --rm ghcr.io/vigilant-llc/runner-guard:latest --version || echo \"Warning: runner-guard version check failed\"\n\necho \"Static analysis tools verification complete\"\n" - - name: Run compile with security tools - run: "set -e\necho \"Running gh aw compile with security tools to download Docker images...\"\n\n# Run compile with all security scanner flags to download Docker images\n# Store the output in a file for inspection\n\"$GITHUB_WORKSPACE/gh-aw\" compile --zizmor --poutine --actionlint --runner-guard 2>&1 | tee /tmp/gh-aw/agent/compile-output.txt\n\necho \"Compile with security tools completed\"\necho \"Output saved to /tmp/gh-aw/agent/compile-output.txt\"" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -558,6 +546,18 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Build gh-aw from source + run: | + set -e + make build + "$GITHUB_WORKSPACE/gh-aw" --version + - name: Pull static analysis Docker images + run: "set -e\necho \"Pulling Docker images for static analysis tools...\"\n\n# Pull zizmor Docker image\necho \"Pulling zizmor image...\"\ndocker pull ghcr.io/zizmorcore/zizmor:latest\n\n# Pull poutine Docker image\necho \"Pulling poutine image...\"\ndocker pull ghcr.io/boostsecurityio/poutine:latest\n\n# Pull runner-guard Docker image\necho \"Pulling runner-guard image...\"\ndocker pull ghcr.io/vigilant-llc/runner-guard:latest\n\necho \"All static analysis Docker images pulled successfully\"\n" + - name: Verify static analysis tools + run: "set -e\necho \"Verifying static analysis tools are available...\"\n\n# Verify zizmor\necho \"Testing zizmor...\"\ndocker run --rm ghcr.io/zizmorcore/zizmor:latest --version || echo \"Warning: zizmor version check failed\"\n\n# Verify poutine\necho \"Testing poutine...\"\ndocker run --rm ghcr.io/boostsecurityio/poutine:latest --version || echo \"Warning: poutine version check failed\"\n\n# Verify runner-guard\necho \"Testing runner-guard...\"\ndocker run --rm ghcr.io/vigilant-llc/runner-guard:latest --version || echo \"Warning: runner-guard version check failed\"\n\necho \"Static analysis tools verification complete\"\n" + - name: Run compile with security tools + run: "set -e\necho \"Running gh aw compile with security tools to download Docker images...\"\n\n# Run compile with all security scanner flags to download Docker images\n# Store the output in a file for inspection\n\"$GITHUB_WORKSPACE/gh-aw\" compile --zizmor --poutine --actionlint --runner-guard 2>&1 | tee /tmp/gh-aw/agent/compile-output.txt\n\necho \"Compile with security tools completed\"\necho \"Output saved to /tmp/gh-aw/agent/compile-output.txt\"" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/step-name-alignment.lock.yml b/.github/workflows/step-name-alignment.lock.yml index f325052f708..91e49147be8 100644 --- a/.github/workflows/step-name-alignment.lock.yml +++ b/.github/workflows/step-name-alignment.lock.yml @@ -478,9 +478,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Build step alignment manifest - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n\nMANIFEST_JSONL=\"/tmp/gh-aw/agent/step-alignment-input.jsonl\"\nMANIFEST_JSON=\"/tmp/gh-aw/agent/step-alignment-input.json\"\n: > \"$MANIFEST_JSONL\"\n\nwhile IFS= read -r workflow_file; do\n yq -o=json \\\n '.jobs | to_entries[] | .value.steps[]? | {\"workflow_file\": \"'\"$workflow_file\"'\", \"step_name\": (.name // \"\"), \"action_uses\": (.uses // \"\")}' \\\n \"$workflow_file\" >> \"$MANIFEST_JSONL\"\ndone < <(find .github/workflows -name \"*.lock.yml\" -type f | sort)\n\njq -s '.' \"$MANIFEST_JSONL\" > \"$MANIFEST_JSON\"\nrm -f \"$MANIFEST_JSONL\"\n\necho \"Wrote $(jq 'length' \"$MANIFEST_JSON\") step records to $MANIFEST_JSON\"" - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -497,6 +494,9 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Build step alignment manifest + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n\nMANIFEST_JSONL=\"/tmp/gh-aw/agent/step-alignment-input.jsonl\"\nMANIFEST_JSON=\"/tmp/gh-aw/agent/step-alignment-input.json\"\n: > \"$MANIFEST_JSONL\"\n\nwhile IFS= read -r workflow_file; do\n yq -o=json \\\n '.jobs | to_entries[] | .value.steps[]? | {\"workflow_file\": \"'\"$workflow_file\"'\", \"step_name\": (.name // \"\"), \"action_uses\": (.uses // \"\")}' \\\n \"$workflow_file\" >> \"$MANIFEST_JSONL\"\ndone < <(find .github/workflows -name \"*.lock.yml\" -type f | sort)\n\njq -s '.' \"$MANIFEST_JSONL\" > \"$MANIFEST_JSON\"\nrm -f \"$MANIFEST_JSONL\"\n\necho \"Wrote $(jq 'length' \"$MANIFEST_JSON\") step records to $MANIFEST_JSON\"" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/super-linter.lock.yml b/.github/workflows/super-linter.lock.yml index 6add0fffd0a..8b8f7a56bf8 100644 --- a/.github/workflows/super-linter.lock.yml +++ b/.github/workflows/super-linter.lock.yml @@ -490,12 +490,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Download super-linter log - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: super-linter-log - path: /tmp/gh-aw/agent/ - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -512,6 +506,12 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Download super-linter log + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: super-linter-log + path: /tmp/gh-aw/agent/ + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index db021e767e3..d6bd964e0ba 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -520,15 +520,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Install dependencies - run: npm ci - working-directory: ./docs - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build documentation - run: npm run build - working-directory: ./docs - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -555,6 +546,15 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: false run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Install dependencies + run: npm ci + working-directory: ./docs + - env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Build documentation + run: npm run build + working-directory: ./docs + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/unbloat-docs.lock.yml b/.github/workflows/unbloat-docs.lock.yml index c57e9abeb01..28be7d350c9 100644 --- a/.github/workflows/unbloat-docs.lock.yml +++ b/.github/workflows/unbloat-docs.lock.yml @@ -536,6 +536,22 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + id: restore_cache_memory_0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -558,22 +574,6 @@ jobs: npx astro build working-directory: ./docs - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - id: restore_cache_memory_0 - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index 5a56069467a..6226bc92857 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -581,6 +581,16 @@ jobs: DIFC_PROXY_IMAGE: 'ghcr.io/github/gh-aw-mcpg:v0.3.33' run: | bash "${RUNNER_TEMP}/gh-aw/actions/start_difc_proxy.sh" + # Repo memory git-based storage configuration from frontmatter processed below + - name: Clone wiki-memory branch (default) + env: + GH_TOKEN: ${{ github.token }} + GITHUB_SERVER_URL: ${{ github.server_url }} + BRANCH_NAME: master + TARGET_REPO: ${{ github.repository }}.wiki + MEMORY_DIR: /tmp/gh-aw/repo-memory/default + CREATE_ORPHAN: false + run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Pre-fetch release and merged PR data if: ${{ needs.activation.outputs.prefetch_strategy == 'eager' }} run: |- @@ -607,16 +617,6 @@ jobs: GITHUB_GRAPHQL_URL: https://localhost:18443/api/graphql GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NODE_EXTRA_CA_CERTS: /tmp/gh-aw/proxy-logs/proxy-tls/ca.crt - # Repo memory git-based storage configuration from frontmatter processed below - - name: Clone wiki-memory branch (default) - env: - GH_TOKEN: ${{ github.token }} - GITHUB_SERVER_URL: ${{ github.server_url }} - BRANCH_NAME: master - TARGET_REPO: ${{ github.repository }}.wiki - MEMORY_DIR: /tmp/gh-aw/repo-memory/default - CREATE_ORPHAN: false - run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/weekly-issue-summary.lock.yml b/.github/workflows/weekly-issue-summary.lock.yml index d224323bd3e..1ee4060a6cb 100644 --- a/.github/workflows/weekly-issue-summary.lock.yml +++ b/.github/workflows/weekly-issue-summary.lock.yml @@ -493,21 +493,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - name: Setup Python environment - run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - - name: Install Python scientific libraries - run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: python-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - # Cache memory file share configuration from frontmatter processed below - name: Create cache-memory directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" @@ -524,6 +509,21 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: approved run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Setup Python environment + run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" + - name: Install Python scientific libraries + run: "# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/agent/venv ]; then\n python3 -m venv /tmp/gh-aw/agent/venv\nfi\necho \"/tmp/gh-aw/agent/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/agent/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n\n# Verify installations\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import numpy; print(f'NumPy {numpy.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import pandas; print(f'Pandas {pandas.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import matplotlib; print(f'Matplotlib {matplotlib.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import seaborn; print(f'Seaborn {seaborn.__version__} installed')\"\n/tmp/gh-aw/agent/venv/bin/python3 -c \"import scipy; print(f'SciPy {scipy.__version__} installed')\"\n\necho \"All scientific libraries installed successfully\"\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: python-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/workflow-health-manager.lock.yml b/.github/workflows/workflow-health-manager.lock.yml index cf708b497f1..4bbdd20fca5 100644 --- a/.github/workflows/workflow-health-manager.lock.yml +++ b/.github/workflows/workflow-health-manager.lock.yml @@ -486,11 +486,6 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build Inventory - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n# Run compilation validation and capture output\ngh aw compile --validate > /tmp/gh-aw/agent/compile-validate.txt 2>&1 || true\n# List executable workflow files (exclude shared/ subdirectory)\nls .github/workflows/*.md 2>/dev/null > /tmp/gh-aw/agent/workflow-list.txt || true\necho \"Inventory complete: $(wc -l < /tmp/gh-aw/agent/workflow-list.txt | tr -d ' ') workflows found\"" - # Repo memory git-based storage configuration from frontmatter processed below - name: Clone repo-memory branch (default) env: @@ -501,6 +496,11 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Build Inventory + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n# Run compilation validation and capture output\ngh aw compile --validate > /tmp/gh-aw/agent/compile-validate.txt 2>&1 || true\n# List executable workflow files (exclude shared/ subdirectory)\nls .github/workflows/*.md 2>/dev/null > /tmp/gh-aw/agent/workflow-list.txt || true\necho \"Inventory complete: $(wc -l < /tmp/gh-aw/agent/workflow-list.txt | tr -d ' ') workflows found\"" + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/pkg/workflow/compiler_yaml_main_job.go b/pkg/workflow/compiler_yaml_main_job.go index 4aaadf633be..869bea2f180 100644 --- a/pkg/workflow/compiler_yaml_main_job.go +++ b/pkg/workflow/compiler_yaml_main_job.go @@ -190,8 +190,10 @@ func (c *Compiler) generateInitialAndCheckoutSteps(yaml *strings.Builder, data * } // generateRuntimeAndWorkspaceSetupSteps emits runtime setup steps, the gh-aw temp directory -// creation step, GitHub Enterprise CLI configuration, DIFC proxy start, custom steps, cache -// steps, cache-memory steps, and repo-memory steps. +// creation step, GitHub Enterprise CLI configuration, DIFC proxy start, cache steps, +// cache-memory steps, repo-memory steps, and then the user's custom steps. +// Memory restore steps (cache-memory, repo-memory) intentionally run before custom steps so +// that deterministic steps: code can read memory without requiring an LLM turn. // It mutates data.CustomSteps (via deduplication) and returns whether the custom steps // themselves contain a checkout action (used by the caller to compute needsGitConfig). func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, data *WorkflowData, needsCheckout bool) bool { @@ -224,20 +226,22 @@ func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, // integrity filtering before the agent runs. Must start before custom steps. c.generateStartDIFCProxyStep(yaml, data) - c.emitCustomSteps(yaml, data, customStepsContainCheckout, runtimeSetupSteps) - // Add cache steps if cache configuration is present compilerYamlLog.Printf("Generating cache steps for workflow") generateCacheSteps(yaml, data, c.verbose) - // Add cache-memory steps if cache-memory configuration is present + // Add cache-memory steps before custom steps so that user steps: code can read + // /tmp/gh-aw/cache-memory// without an LLM turn. compilerYamlLog.Printf("Generating cache-memory steps for workflow") generateCacheMemorySteps(yaml, data) - // Add repo-memory clone steps if repo-memory configuration is present + // Add repo-memory clone steps before custom steps so that user steps: code can read + // /tmp/gh-aw/repo-memory// without an LLM turn. compilerYamlLog.Printf("Generating repo-memory steps for workflow") generateRepoMemorySteps(yaml, data) + c.emitCustomSteps(yaml, data, customStepsContainCheckout, runtimeSetupSteps) + return customStepsContainCheckout } diff --git a/pkg/workflow/compiler_yaml_main_job_test.go b/pkg/workflow/compiler_yaml_main_job_test.go index ae50e5f4c50..f1730be765a 100644 --- a/pkg/workflow/compiler_yaml_main_job_test.go +++ b/pkg/workflow/compiler_yaml_main_job_test.go @@ -1006,3 +1006,55 @@ func TestGenerateMainJobStepsRestoreActionsFolder(t *testing.T) { assert.NotContains(t, result, "Restore actions folder", "agent job should NOT have restore step in action mode") }) } + +// TestMemoryRestoreStepsOrderBeforeCustomSteps asserts that cache-memory and repo-memory +// restore steps are emitted before user custom steps so that deterministic steps: code +// can read memory without requiring an LLM turn. +func TestMemoryRestoreStepsOrderBeforeCustomSteps(t *testing.T) { + compiler := NewCompiler() + compiler.stepOrderTracker = NewStepOrderTracker() + + data := &WorkflowData{ + Name: "Test Workflow", + AI: "copilot", + MarkdownContent: "Test prompt", + EngineConfig: &EngineConfig{ID: "copilot"}, + ParsedTools: &ToolsConfig{}, + CustomSteps: `steps: + - name: My Custom Step + run: echo "hello"`, + CacheMemoryConfig: &CacheMemoryConfig{ + Caches: []CacheMemoryEntry{ + {ID: "default", Key: "memory-test-${{ github.run_id }}"}, + }, + }, + RepoMemoryConfig: &RepoMemoryConfig{ + Memories: []RepoMemoryEntry{ + { + ID: "default", + BranchName: "memory/default", + }, + }, + }, + } + + var buf strings.Builder + err := compiler.generateMainJobSteps(&buf, data) + require.NoError(t, err, "generateMainJobSteps should not error") + + result := buf.String() + + // Locate the positions of the key markers in the generated YAML. + cacheMemoryPos := strings.Index(result, "Cache memory file share") + repoMemoryPos := strings.Index(result, "Clone repo-memory branch") + customStepPos := strings.Index(result, "My Custom Step") + + require.Greater(t, cacheMemoryPos, -1, "cache-memory setup comment should be present in output") + require.Greater(t, repoMemoryPos, -1, "repo-memory clone step should be present in output") + require.Greater(t, customStepPos, -1, "custom step should be present in output") + + assert.Less(t, cacheMemoryPos, customStepPos, + "cache-memory restore steps must appear before custom steps") + assert.Less(t, repoMemoryPos, customStepPos, + "repo-memory restore steps must appear before custom steps") +}