diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 2f79bbb9ec1..92816fb0b94 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ffd9ec0077f331d4c06c7b78f8beff4675585cc50ce300ee5b712a60e223b056","body_hash":"f9e809306104ce48e0df05b6b14f890f1a239abb6e01ac2f5d3a90384e205793","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0f0ef0a4aa8d28aafb55804f9cd24ca178434b666329bd521e5b4fd0b98fe915","body_hash":"1021d8e5f0f5f45cff34282cb573c3f40cf95f80095d978746133ca19d7fd36e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -397,7 +397,6 @@ jobs: needs: - activation - collect_anthropic_models - - collect_copilot_api_models - collect_copilot_billing_models - collect_copilot_sdk_models - collect_gemini_models @@ -1117,6 +1116,10 @@ jobs: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} shell: bash + - name: Log raw response + if: always() + run: cat /tmp/gh-aw/agent/model-inventory/anthropic/raw.json 2>/dev/null || echo "(no raw.json)" + shell: bash - name: Upload Anthropic artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -1128,48 +1131,6 @@ jobs: /tmp/gh-aw/agent/model-inventory/anthropic/raw.json retention-days: 7 - collect_copilot_api_models: - needs: activation - runs-on: ubuntu-latest - permissions: - copilot-requests: write - steps: - - name: Configure GH_HOST for enterprise compatibility - id: ghes-host-config - shell: bash - run: | # zizmor: ignore[github-env] - GITHUB_SERVER_URL is set by GitHub Actions, not user input. - # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct - # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. - GH_HOST="${GITHUB_SERVER_URL#https://}" - GH_HOST="${GH_HOST#http://}" - echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" - - name: Fetch GitHub Copilot API models - id: fetch - run: | - set -euo pipefail - OUT="/tmp/gh-aw/agent/model-inventory/copilot-api" - mkdir -p "$OUT" - HTTP_STATUS=$(curl -sf -o "$OUT/raw.json" -w "%{http_code}" \ - -H "Authorization: ******" \ - https://api.githubcopilot.com/models) || true - if [ "${HTTP_STATUS:-0}" = "200" ]; then - echo "status=ok" >> "$GITHUB_OUTPUT" - else - echo "{\"error\":\"HTTP $HTTP_STATUS\"}" > "$OUT/raw.json" - echo "status=error" >> "$GITHUB_OUTPUT" - fi - env: - GITHUB_TOKEN: ${{ github.token }} - shell: bash - - name: Upload Copilot API models artifact - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: copilot-api-models - path: /tmp/gh-aw/agent/model-inventory/copilot-api/raw.json - retention-days: 7 - collect_copilot_billing_models: needs: activation runs-on: ubuntu-latest @@ -1201,6 +1162,10 @@ jobs: > "$OUT/models.json" echo "status=placeholder" >> "$GITHUB_OUTPUT" shell: bash + - name: Log placeholder + if: always() + run: cat /tmp/gh-aw/agent/model-inventory/copilot-billing/models.json 2>/dev/null || echo "(no models.json)" + shell: bash - name: Upload Copilot billing models artifact if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -1257,6 +1222,10 @@ jobs: env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} shell: bash + - name: Log models + if: always() + run: cat /tmp/gh-aw/agent/model-inventory/copilot-sdk/models.json 2>/dev/null || echo "(no models.json)" + shell: bash - name: Upload Copilot SDK models artifact if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -1318,6 +1287,10 @@ jobs: env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} shell: bash + - name: Log raw response + if: always() + run: cat /tmp/gh-aw/agent/model-inventory/gemini/raw.json 2>/dev/null || echo "(no raw.json)" + shell: bash - name: Upload Gemini artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -1378,6 +1351,10 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} shell: bash + - name: Log raw response + if: always() + run: cat /tmp/gh-aw/agent/model-inventory/openai/raw.json 2>/dev/null || echo "(no raw.json)" + shell: bash - name: Upload OpenAI artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -1394,7 +1371,6 @@ jobs: - activation - agent - collect_anthropic_models - - collect_copilot_api_models - collect_copilot_billing_models - collect_copilot_sdk_models - collect_gemini_models diff --git a/.github/workflows/daily-model-inventory.md b/.github/workflows/daily-model-inventory.md index 78d595c4b33..b4033af46c2 100644 --- a/.github/workflows/daily-model-inventory.md +++ b/.github/workflows/daily-model-inventory.md @@ -84,6 +84,11 @@ jobs: echo "status=error" >> "$GITHUB_OUTPUT" fi + - name: Log raw response + if: always() + shell: bash + run: cat /tmp/gh-aw/agent/model-inventory/openai/raw.json 2>/dev/null || echo "(no raw.json)" + - name: Upload OpenAI artifacts if: always() uses: actions/upload-artifact@v7.0.1 @@ -138,6 +143,11 @@ jobs: echo "status=error" >> "$GITHUB_OUTPUT" fi + - name: Log raw response + if: always() + shell: bash + run: cat /tmp/gh-aw/agent/model-inventory/anthropic/raw.json 2>/dev/null || echo "(no raw.json)" + - name: Upload Anthropic artifacts if: always() uses: actions/upload-artifact@v7.0.1 @@ -193,6 +203,11 @@ jobs: echo "status=error" >> "$GITHUB_OUTPUT" fi + - name: Log raw response + if: always() + shell: bash + run: cat /tmp/gh-aw/agent/model-inventory/gemini/raw.json 2>/dev/null || echo "(no raw.json)" + - name: Upload Gemini artifacts if: always() uses: actions/upload-artifact@v7.0.1 @@ -227,46 +242,17 @@ jobs: > "$OUT/models.json" echo "status=placeholder" >> "$GITHUB_OUTPUT" - - name: Upload Copilot billing models artifact + - name: Log placeholder if: always() - uses: actions/upload-artifact@v7.0.1 - with: - name: copilot-billing-models - path: /tmp/gh-aw/agent/model-inventory/copilot-billing/models.json - if-no-files-found: error - retention-days: 7 - - collect_copilot_api_models: - runs-on: ubuntu-latest - needs: [activation] - permissions: - copilot-requests: write - steps: - - name: Fetch GitHub Copilot API models - id: fetch shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - OUT="/tmp/gh-aw/agent/model-inventory/copilot-api" - mkdir -p "$OUT" - HTTP_STATUS=$(curl -sf -o "$OUT/raw.json" -w "%{http_code}" \ - -H "Authorization: ******" \ - https://api.githubcopilot.com/models) || true - if [ "${HTTP_STATUS:-0}" = "200" ]; then - echo "status=ok" >> "$GITHUB_OUTPUT" - else - echo "{\"error\":\"HTTP $HTTP_STATUS\"}" > "$OUT/raw.json" - echo "status=error" >> "$GITHUB_OUTPUT" - fi + run: cat /tmp/gh-aw/agent/model-inventory/copilot-billing/models.json 2>/dev/null || echo "(no models.json)" - - name: Upload Copilot API models artifact + - name: Upload Copilot billing models artifact if: always() uses: actions/upload-artifact@v7.0.1 with: - name: copilot-api-models - path: /tmp/gh-aw/agent/model-inventory/copilot-api/raw.json + name: copilot-billing-models + path: /tmp/gh-aw/agent/model-inventory/copilot-billing/models.json if-no-files-found: error retention-days: 7 @@ -310,6 +296,11 @@ jobs: } EOF + - name: Log models + if: always() + shell: bash + run: cat /tmp/gh-aw/agent/model-inventory/copilot-sdk/models.json 2>/dev/null || echo "(no models.json)" + - name: Upload Copilot SDK models artifact if: always() uses: actions/upload-artifact@v7.0.1 @@ -383,16 +374,13 @@ updating. ## Inputs -The pre-job steps have already fetched model lists from OpenAI, Anthropic, Gemini, and the GitHub Copilot API, then merged +The pre-job steps have already fetched model lists from OpenAI, Anthropic, and Gemini, then merged them into: - Combined inventory: `/tmp/gh-aw/agent/model-inventory/inventory.json` - Individual provider files: `/tmp/gh-aw/agent/model-inventory/artifacts/-models/models.json` - Raw provider responses: `/tmp/gh-aw/agent/model-inventory/artifacts/-models/raw.json` - Predownloaded models.dev API index: `/tmp/gh-aw/agent/model-inventory/models-dev/api.json` -- GitHub Copilot API models: `/tmp/gh-aw/agent/model-inventory/artifacts/copilot-api-models/raw.json` - (fetched from `https://api.githubcopilot.com/models`; raw API response — if the file contains an `error` field, - treat Copilot API data as unavailable for this run) - Copilot SDK models: `/tmp/gh-aw/agent/model-inventory/artifacts/copilot-sdk-models/models.json` (fetched through the Copilot CLI SDK; use each model's `billing` fields for pricing validation) - Copilot live provider metadata: `/tmp/gh-aw/agent/model-inventory/reflect.json` (generated in @@ -598,21 +586,12 @@ The Copilot pricing table is now in: This file was populated in Step 2.5 from: `https://docs.github.com/en/copilot/reference/copilot-billing/models-and-pricing`. -The GitHub Copilot API response is available at: - -- `/tmp/gh-aw/agent/model-inventory/artifacts/copilot-api-models/raw.json` - -This file is the raw response fetched from `https://api.githubcopilot.com/models`. If the file contains an `error` field, -treat it as unavailable and skip its use as a validation source for this run. - The Copilot SDK models are available at: - `/tmp/gh-aw/agent/model-inventory/artifacts/copilot-sdk-models/models.json` -Use the Copilot SDK (`billing` fields), reflect endpoint (`billing.multiplier`), Copilot API endpoint -(`billing.multiplier`), and docs pricing table as validation sources for `models.json` pricing fields. -Prefer Copilot SDK data when available; cross-check it against reflect data, use the Copilot API for -discovering models absent from the other sources, and use docs table values as a tertiary cross-check. +Use the Copilot SDK (`billing` fields), reflect endpoint (`billing.multiplier`), and docs pricing table as validation sources for `models.json` pricing fields. +Prefer Copilot SDK data when available; cross-check it against reflect data, and use docs table values as a secondary cross-check. Also validate Copilot SDK routing metadata in `models.json` for `github-copilot` models: - `provider_type` (for SDK provider selection) @@ -622,7 +601,7 @@ When updating `models.json`, preserve or add `wire_api` for Copilot models where provides it. Keep `wire_api` absent for models/providers where it is not applicable. Treat `gpt-4o-mini`, `gpt-4.1`, `gpt-4o`, and `gpt-5.4-nano` as intentionally deprecated -Copilot-facing model IDs. Keep ignoring them even if they appear in the reflect data, Copilot API, +Copilot-facing model IDs. Keep ignoring them even if they appear in the reflect data, docs table, `models.dev`, or live provider inventories: do not propose adding or restoring them in `pkg/cli/data/models.json` (or `actions/setup/js/models.json`), and exclude them from missing/discrepancy tables. @@ -634,7 +613,7 @@ When pricing updates are required, first run: make refresh-models-json ``` -Then validate/adjust pricing entries against Copilot SDK, reflect, Copilot API, and docs-derived data. +Then validate/adjust pricing entries against Copilot SDK, reflect, and docs-derived data. For each provider's enriched data, validate pricing/model coverage for each model: @@ -647,25 +626,18 @@ For each provider's enriched data, validate pricing/model coverage for each mode `/tmp/gh-aw/agent/model-inventory/reflect.json`. For each model, use the `billing.multiplier` field to cross-check the SDK multiplier. Compare against the matching `github/` entry in `models.json`, and list discrepancies or missing models. - Cross-reference against the Copilot API and docs table as secondary validation sources. - -3. **GitHub Copilot API** — use the `copilot-api` models from - `/tmp/gh-aw/agent/model-inventory/artifacts/copilot-api-models/raw.json` as a supplementary - source. For each model in `.data[]`, use `billing.multiplier` where present as a secondary cross-check against - the SDK and reflect data. Flag any model IDs in the Copilot API response that are not in the SDK or reflect data, - or not in `models.json` — these may be newly available models. Skip this source if the file - contains an `error` field. + Cross-reference against the docs table as a secondary validation source. -4. **Gemini API** — use `inputTokenLimit` / `outputTokenLimit` as an approximate proxy for model +3. **Gemini API** — use `inputTokenLimit` / `outputTokenLimit` as an approximate proxy for model complexity (this is an inference heuristic, not a definitive billing mapping). Large-context, high-output-limit models typically correspond to higher-priced tiers; smaller Flash models to lower-priced tiers. Flag any models whose limits suggest a pricing-tier change versus what is currently in `models.json`. -5. **OpenAI API** — use `owned_by` and model-ID naming conventions (e.g. `-mini`, `-nano`, `o1`, +4. **OpenAI API** — use `owned_by` and model-ID naming conventions (e.g. `-mini`, `-nano`, `o1`, `o3`) to cross-check current pricing tiers. Flag missing models or likely mismatches. -6. **Anthropic API** — use `display_name` family grouping (haiku/sonnet/opus) to validate +5. **Anthropic API** — use `display_name` family grouping (haiku/sonnet/opus) to validate current pricing tiers. Flag any new model IDs not yet in `models.json`. Produce a consolidated pricing gap table listing: @@ -723,7 +695,7 @@ If you found any meaningful updates to propose, create a GitHub issue using `cre Brief description of what was found. -- Providers queried: OpenAI, Anthropic, Gemini, Copilot SDK, Copilot (reflect), Copilot API +- Providers queried: OpenAI, Anthropic, Gemini, Copilot SDK, Copilot (reflect) - Total models found: - Proposed alias changes: - Pricing gaps found: @@ -737,7 +709,6 @@ Brief description of what was found. | gemini | 28 | ✅ ok | | copilot-sdk | 35 | ✅ ok | | copilot (reflect) | 35 | ✅ ok | -| copilot-api | 38 | ✅ ok | ### Raw API Fields Discovered