diff --git a/.gitattributes b/.gitattributes index 4accb6fe..d1c2923e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,4 @@ *.mp4 filter=lfs diff=lfs merge=lfs -text *.webm filter=lfs diff=lfs merge=lfs -text *.svg filter=lfs diff=lfs merge=lfs -text +docs/favicon.svg !filter !diff !merge diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml new file mode 100644 index 00000000..1e44b97b --- /dev/null +++ b/.github/workflows/fix-drift.yml @@ -0,0 +1,128 @@ +name: Fix Drift +on: + workflow_dispatch: + workflow_run: + workflows: ["Drift Tests"] + types: [completed] + branches: [main] + +concurrency: + group: drift-fix + cancel-in-progress: false + +jobs: + fix: + if: >- + github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + pull-requests: write + issues: write + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + + # Step 0: Configure git identity and create fix branch + - name: Configure git + run: | + git config user.name "llmock-drift-bot" + git config user.email "drift-bot@copilotkit.ai" + git checkout -B fix/drift-$(date +%Y-%m-%d)-${{ github.run_id }} + + # Step 1: Detect drift and produce report + - name: Collect drift report + id: detect + run: | + set +e + npx tsx scripts/drift-report-collector.ts + EXIT_CODE=$? + set -e + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + if [ "$EXIT_CODE" -eq 2 ]; then + : # critical drift found, continue + elif [ "$EXIT_CODE" -ne 0 ]; then + echo "::error::Collector script crashed with exit code $EXIT_CODE" + exit $EXIT_CODE + fi + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + + # Always upload the report as an artifact + - name: Upload drift report + if: always() + uses: actions/upload-artifact@v4 + with: + name: drift-report + path: drift-report.json + if-no-files-found: warn + retention-days: 30 + + # Step 2: Exit if no critical drift + - name: Check for critical diffs + id: check + env: + DETECT_EXIT_CODE: ${{ steps.detect.outputs.exit_code }} + run: | + if [ "$DETECT_EXIT_CODE" = "2" ]; then + echo "skip=false" >> $GITHUB_OUTPUT + echo "Critical drift detected" + else + echo "skip=true" >> $GITHUB_OUTPUT + echo "No critical drift detected (exit code: $DETECT_EXIT_CODE) — skipping fix" + fi + + # Step 3: Invoke Claude Code to fix + - name: Auto-fix drift + if: steps.check.outputs.skip != 'true' + run: npx tsx scripts/fix-drift.ts + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + + # Upload Claude Code output for debugging + - name: Upload Claude Code logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: claude-code-output + path: claude-code-output.log + if-no-files-found: warn + retention-days: 30 + + # Step 4: Verify fix independently + - name: Verify conformance + if: steps.check.outputs.skip != 'true' + run: pnpm test + + - name: Verify drift resolved + if: steps.check.outputs.skip != 'true' + run: pnpm test:drift + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + + # Step 5: Create PR on success + - name: Create PR + if: success() && steps.check.outputs.skip != 'true' + run: npx tsx scripts/fix-drift.ts --create-pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Step 6: Open issue on failure + - name: Create issue on failure + if: failure() && steps.check.outputs.skip != 'true' + run: npx tsx scripts/fix-drift.ts --create-issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index 6882bcd9..b76d6d1b 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -6,6 +6,7 @@ on: jobs: drift: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -14,8 +15,35 @@ jobs: node-version: 22 cache: pnpm - run: pnpm install --frozen-lockfile - - run: pnpm test:drift + + - name: Run drift tests + id: drift + run: | + set +e + npx tsx scripts/drift-report-collector.ts + EXIT_CODE=$? + set -e + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + if [ "$EXIT_CODE" -eq 2 ]; then + : # critical drift found, continue + elif [ "$EXIT_CODE" -ne 0 ]; then + echo "::error::Collector script crashed with exit code $EXIT_CODE" + exit $EXIT_CODE + fi env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + + - name: Upload drift report + if: always() + uses: actions/upload-artifact@v4 + with: + name: drift-report + path: drift-report.json + if-no-files-found: warn + retention-days: 30 + + - name: Fail if critical drift detected + if: steps.drift.outputs.exit_code == '2' + run: exit 1 diff --git a/CLAUDE.md b/CLAUDE.md index be295bfd..2ba92b46 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,15 @@ entire repo, not just staged files. - When adding features or fixing bugs, add or update tests - Run `pnpm test` before pushing +## Drift Remediation + +Automated drift remediation lives in `scripts/`: + +- `scripts/drift-report-collector.ts` — runs drift tests, produces `drift-report.json` +- `scripts/fix-drift.ts` — reads drift report, invokes Claude Code to fix builders, creates PR or issue + +See `DRIFT.md` for full documentation and `.github/workflows/fix-drift.yml` for the CI workflow. + ## Commit Messages - This repo enforces conventional commit prefixes via commitlint: `fix:`, `feat:`, `docs:`, `test:`, `chore:`, `refactor:`, etc. diff --git a/DRIFT.md b/DRIFT.md index 569abf68..b8a0ffb2 100644 --- a/DRIFT.md +++ b/DRIFT.md @@ -106,7 +106,7 @@ When a model is deprecated: ## WebSocket Drift Coverage -In addition to the 19 existing drift tests (16 HTTP response-shape + 3 model deprecation), WebSocket drift tests cover llmock's WS protocols: +In addition to the 19 existing drift tests (16 HTTP response-shape + 3 model deprecation), WebSocket drift tests cover llmock's WS protocols (4 verified + 2 canary = 6 WS tests): | Protocol | Text | Tool Call | Real Endpoint | Status | | ------------------- | ---- | --------- | ------------------------------------------------------------------- | ---------- | @@ -138,6 +138,29 @@ Drift tests run on a schedule: See `.github/workflows/test-drift.yml`. +## Automated Drift Remediation + +When the daily drift test detects critical diffs on the `main` branch, the `fix-drift.yml` workflow runs automatically: + +1. **Collect** — `scripts/drift-report-collector.ts` runs drift tests and produces a structured `drift-report.json` +2. **Fix** — `scripts/fix-drift.ts` (default mode) constructs a prompt from the report and invokes Claude Code to fix the builders +3. **Verify** — Independent `pnpm test` and `pnpm test:drift` steps confirm the fix works +4. **PR** — `scripts/fix-drift.ts --create-pr` stages and commits the changes, bumps the version, and opens a pull request +5. **Issue** (on failure) — `scripts/fix-drift.ts --create-issue` opens a GitHub issue with the drift report and Claude Code output + +Steps 2 and 4/5 are separate invocations of `fix-drift.ts` with different modes. + +### Artifacts + +Both workflows upload artifacts: + +- `drift-report.json` — structured drift data (retained 30 days) +- `claude-code-output.log` — Claude Code's reasoning and tool calls (fix workflow only) + +### Manual trigger + +The fix workflow also supports `workflow_dispatch` for manual runs. + ## Cost -~25 API calls per run (16 HTTP response-shape + 3 model listing + 4 WS + 2 canaries) using the cheapest available models (`gpt-4o-mini`, `gpt-4o-mini-realtime-preview`, `claude-haiku-4-5-20251001`, `gemini-2.5-flash`) with 10-100 max tokens each. Under $0.15/week at daily cadence. When Gemini Live text-capable models become available, this will increase to 6 WS calls. +~25 API calls per run (16 HTTP response-shape + 3 model listing + 6 WS including canaries) using the cheapest available models (`gpt-4o-mini`, `gpt-4o-mini-realtime-preview`, `claude-haiku-4-5-20251001`, `gemini-2.5-flash`) with 10-100 max tokens each. Under $0.15/week at daily cadence. When Gemini Live text-capable models become available, the 2 canary tests will become full drift tests, increasing real WS connections from 4 to 6. diff --git a/docs/favicon.svg b/docs/favicon.svg index 93121b94..63285eaf 100644 --- a/docs/favicon.svg +++ b/docs/favicon.svg @@ -1,30 +1,3 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +version https://git-lfs.github.com/spec/v1 +oid sha256:4a218f7047973946fe28120c9209e2873144118d5b5a7e2ea9e7aa4c407559fb +size 3265 diff --git a/package.json b/package.json index e2002ba7..d6f83f00 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,9 @@ "typescript-eslint": "^8.35.1", "@anthropic-ai/sdk": "^0.78.0", "@google/generative-ai": "^0.24.0", + "@types/node": "^22.0.0", "openai": "^4.0.0", + "tsx": "^4.19.0", "vitest": "^3.2.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1b8931b5..fa161766 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,7 +16,7 @@ importers: version: 0.17.4 '@commitlint/cli': specifier: ^19.8.1 - version: 19.8.1(@types/node@25.3.3)(typescript@5.9.3) + version: 19.8.1(@types/node@22.19.15)(typescript@5.9.3) '@commitlint/config-conventional': specifier: ^19.8.0 version: 19.8.1 @@ -26,6 +26,9 @@ importers: '@google/generative-ai': specifier: ^0.24.0 version: 0.24.1 + '@types/node': + specifier: ^22.0.0 + version: 22.19.15 eslint: specifier: ^9.30.0 version: 9.39.3(jiti@2.6.1) @@ -50,6 +53,9 @@ importers: tsdown: specifier: ^0.12.5 version: 0.12.9(publint@0.3.18)(typescript@5.9.3) + tsx: + specifier: ^4.19.0 + version: 4.21.0 typescript: specifier: ^5.8.3 version: 5.9.3 @@ -58,7 +64,7 @@ importers: version: 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.3) vitest: specifier: ^3.2.1 - version: 3.2.4(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2) + version: 3.2.4(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2) packages: @@ -690,8 +696,8 @@ packages: '@types/node@18.19.130': resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} - '@types/node@25.3.3': - resolution: {integrity: sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==} + '@types/node@22.19.15': + resolution: {integrity: sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==} '@typescript-eslint/eslint-plugin@8.56.1': resolution: {integrity: sha512-Jz9ZztpB37dNC+HU2HI28Bs9QXpzCz+y/twHOwhyrIRdbuVDxSytJNDl6z/aAKlaRIwC7y8wJdkBv7FxYGgi0A==} @@ -1894,6 +1900,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -1924,8 +1935,8 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@7.18.2: - resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} unicode-emoji-modifier-base@1.0.0: resolution: {integrity: sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==} @@ -2143,11 +2154,11 @@ snapshots: '@colors/colors@1.5.0': optional: true - '@commitlint/cli@19.8.1(@types/node@25.3.3)(typescript@5.9.3)': + '@commitlint/cli@19.8.1(@types/node@22.19.15)(typescript@5.9.3)': dependencies: '@commitlint/format': 19.8.1 '@commitlint/lint': 19.8.1 - '@commitlint/load': 19.8.1(@types/node@25.3.3)(typescript@5.9.3) + '@commitlint/load': 19.8.1(@types/node@22.19.15)(typescript@5.9.3) '@commitlint/read': 19.8.1 '@commitlint/types': 19.8.1 tinyexec: 1.0.2 @@ -2194,7 +2205,7 @@ snapshots: '@commitlint/rules': 19.8.1 '@commitlint/types': 19.8.1 - '@commitlint/load@19.8.1(@types/node@25.3.3)(typescript@5.9.3)': + '@commitlint/load@19.8.1(@types/node@22.19.15)(typescript@5.9.3)': dependencies: '@commitlint/config-validator': 19.8.1 '@commitlint/execute-rule': 19.8.1 @@ -2202,7 +2213,7 @@ snapshots: '@commitlint/types': 19.8.1 chalk: 5.6.2 cosmiconfig: 9.0.1(typescript@5.9.3) - cosmiconfig-typescript-loader: 6.2.0(@types/node@25.3.3)(cosmiconfig@9.0.1(typescript@5.9.3))(typescript@5.9.3) + cosmiconfig-typescript-loader: 6.2.0(@types/node@22.19.15)(cosmiconfig@9.0.1(typescript@5.9.3))(typescript@5.9.3) lodash.isplainobject: 4.0.6 lodash.merge: 4.6.2 lodash.uniq: 4.5.0 @@ -2571,7 +2582,7 @@ snapshots: '@types/conventional-commits-parser@5.0.2': dependencies: - '@types/node': 25.3.3 + '@types/node': 22.19.15 '@types/deep-eql@4.0.2': {} @@ -2581,16 +2592,16 @@ snapshots: '@types/node-fetch@2.6.13': dependencies: - '@types/node': 25.3.3 + '@types/node': 22.19.15 form-data: 4.0.5 '@types/node@18.19.130': dependencies: undici-types: 5.26.5 - '@types/node@25.3.3': + '@types/node@22.19.15': dependencies: - undici-types: 7.18.2 + undici-types: 6.21.0 '@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.3)': dependencies: @@ -2691,13 +2702,13 @@ snapshots: chai: 5.3.3 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2))': + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.1(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2) + vite: 7.3.1(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2) '@vitest/pretty-format@3.2.4': dependencies: @@ -2916,9 +2927,9 @@ snapshots: meow: 12.1.1 split2: 4.2.0 - cosmiconfig-typescript-loader@6.2.0(@types/node@25.3.3)(cosmiconfig@9.0.1(typescript@5.9.3))(typescript@5.9.3): + cosmiconfig-typescript-loader@6.2.0(@types/node@22.19.15)(cosmiconfig@9.0.1(typescript@5.9.3))(typescript@5.9.3): dependencies: - '@types/node': 25.3.3 + '@types/node': 22.19.15 cosmiconfig: 9.0.1(typescript@5.9.3) jiti: 2.6.1 typescript: 5.9.3 @@ -3783,6 +3794,13 @@ snapshots: tslib@2.8.1: optional: true + tsx@4.21.0: + dependencies: + esbuild: 0.27.3 + get-tsconfig: 4.13.6 + optionalDependencies: + fsevents: 2.3.3 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -3817,7 +3835,7 @@ snapshots: undici-types@5.26.5: {} - undici-types@7.18.2: {} + undici-types@6.21.0: {} unicode-emoji-modifier-base@1.0.0: {} @@ -3829,13 +3847,13 @@ snapshots: validate-npm-package-name@5.0.1: {} - vite-node@3.2.4(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2): + vite-node@3.2.4(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 7.3.1(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2) + vite: 7.3.1(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2) transitivePeerDependencies: - '@types/node' - jiti @@ -3850,7 +3868,7 @@ snapshots: - tsx - yaml - vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2): + vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2): dependencies: esbuild: 0.27.3 fdir: 6.5.0(picomatch@4.0.3) @@ -3859,16 +3877,17 @@ snapshots: rollup: 4.59.0 tinyglobby: 0.2.15 optionalDependencies: - '@types/node': 25.3.3 + '@types/node': 22.19.15 fsevents: 2.3.3 jiti: 2.6.1 + tsx: 4.21.0 yaml: 2.8.2 - vitest@3.2.4(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2): + vitest@3.2.4(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2)) + '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -3886,11 +3905,11 @@ snapshots: tinyglobby: 0.2.15 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 7.3.1(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2) - vite-node: 3.2.4(@types/node@25.3.3)(jiti@2.6.1)(yaml@2.8.2) + vite: 7.3.1(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2) + vite-node: 3.2.4(@types/node@22.19.15)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.3.3 + '@types/node': 22.19.15 transitivePeerDependencies: - jiti - less diff --git a/scripts/drift-report-collector.ts b/scripts/drift-report-collector.ts new file mode 100644 index 00000000..6692a708 --- /dev/null +++ b/scripts/drift-report-collector.ts @@ -0,0 +1,414 @@ +/// + +/** + * Drift Report Collector + * + * Runs the drift test suite via subprocess with JSON reporter, parses the + * structured output, and writes a drift-report.json file that downstream + * scripts can use to construct auto-fix prompts. + * + * Exit codes: + * 0 — no critical diffs found (or no drift at all) + * 2 — at least one critical diff exists + * 1 — script error (unhandled exception) + * + * Usage: + * npx tsx scripts/drift-report-collector.ts [--out drift-report.json] + */ + +import { execSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import type { DriftEntry, DriftReport, DriftSeverity, ParsedDiff } from "./drift-types.js"; + +// --------------------------------------------------------------------------- +// Vitest JSON reporter types (subset we care about) +// --------------------------------------------------------------------------- + +interface VitestJsonResult { + testResults: VitestTestFile[]; +} + +interface VitestTestFile { + assertionResults: VitestAssertion[]; +} + +interface VitestAssertion { + status: string; + ancestorTitles: string[]; + title: string; + failureMessages: string[]; +} + +// --------------------------------------------------------------------------- +// Provider → file mapping +// --------------------------------------------------------------------------- + +interface ProviderMapping { + builderFile: string; + builderFunctions: string[]; + typesFile: string | null; +} + +const OPENAI_CHAT_MAPPING: ProviderMapping = { + builderFile: "src/helpers.ts", + builderFunctions: [ + "buildTextCompletion", + "buildToolCallCompletion", + "buildTextChunks", + "buildToolCallChunks", + ], + typesFile: "src/types.ts", +}; + +const OPENAI_RESPONSES_MAPPING: ProviderMapping = { + builderFile: "src/responses.ts", + builderFunctions: [ + "buildTextResponse", + "buildToolCallResponse", + "buildTextStreamEvents", + "buildToolCallStreamEvents", + ], + typesFile: null, +}; + +const ANTHROPIC_MAPPING: ProviderMapping = { + builderFile: "src/messages.ts", + builderFunctions: [ + "buildClaudeTextResponse", + "buildClaudeToolCallResponse", + "buildClaudeTextStreamEvents", + "buildClaudeToolCallStreamEvents", + ], + typesFile: null, +}; + +const GEMINI_MAPPING: ProviderMapping = { + builderFile: "src/gemini.ts", + builderFunctions: [ + "buildGeminiTextResponse", + "buildGeminiToolCallResponse", + "buildGeminiTextStreamChunks", + "buildGeminiToolCallStreamChunks", + ], + typesFile: null, +}; + +/** + * Maps provider names (from drift test describe blocks) to source files + * and builder function names. The function names are builder functions for + * each provider (internal or exported) — they are included so Claude Code + * can locate them via Read/Grep. + */ +const PROVIDER_MAP: Record = { + "OpenAI Chat": OPENAI_CHAT_MAPPING, + "OpenAI Responses": OPENAI_RESPONSES_MAPPING, + Anthropic: ANTHROPIC_MAPPING, + "Anthropic Claude": ANTHROPIC_MAPPING, + "Google Gemini": GEMINI_MAPPING, + Gemini: GEMINI_MAPPING, + "OpenAI Realtime": { + builderFile: "src/ws-realtime.ts", + builderFunctions: ["handleWebSocketRealtime", "realtimeItemsToMessages"], + typesFile: null, + }, + "OpenAI Responses WS": { + builderFile: "src/ws-responses.ts", + builderFunctions: ["handleWebSocketResponses"], + typesFile: null, + }, + "Gemini Live": { + builderFile: "src/ws-gemini-live.ts", + builderFunctions: ["handleWebSocketGeminiLive"], + typesFile: null, + }, +}; + +const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; + +// --------------------------------------------------------------------------- +// Parse the formatted drift report text from a vitest failure message +// --------------------------------------------------------------------------- + +/** + * Parse a drift report block from raw vitest failure message content. + * + * The input is a raw vitest failureMessages string that may contain error boilerplate. + * The function scans for the API DRIFT DETECTED header and numbered entries. + * + * Expected format within the message (produced by formatDriftReport): + * ``` + * API DRIFT DETECTED: OpenAI Chat (non-streaming text) + * + * 1. [critical] LLMOCK DRIFT — field in SDK + real API but missing from mock + * Path: choices[0].message.refusal + * SDK: null + * Real: null + * Mock: + * ``` + */ +const VALID_SEVERITIES = new Set(["critical", "warning", "info"]); + +function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { + const headerMatch = text.match(/API DRIFT DETECTED:\s*(.+)/); + if (!headerMatch) return null; + + const context = headerMatch[1].trim(); + const diffs: ParsedDiff[] = []; + + // Match numbered entries: " 1. [severity] issue text\n Path:...\n SDK:...\n Real:...\n Mock:..." + const entryPattern = + /\d+\.\s*\[(\w+)\]\s*(.+)\n\s*Path:\s*(.+)\n\s*SDK:\s*(.+)\n\s*Real:\s*(.+)\n\s*Mock:\s*(.+)/g; + + let match: RegExpExecArray | null; + while ((match = entryPattern.exec(text)) !== null) { + const severity = match[1].trim(); + if (!VALID_SEVERITIES.has(severity as DriftSeverity)) { + console.warn( + `parseDriftBlock: unknown severity "${severity}" — skipping entry. ` + + `Known severities: ${[...VALID_SEVERITIES].join(", ")}`, + ); + continue; + } + diffs.push({ + severity: severity as DriftSeverity, + issue: match[2].trim(), + path: match[3].trim(), + expected: match[4].trim(), + real: match[5].trim(), + mock: match[6].trim(), + }); + } + + const expectedCount = (text.match(/\d+\.\s*\[/g) ?? []).length; + if (expectedCount > 0 && diffs.length < expectedCount) { + console.warn(`parseDriftBlock: parsed ${diffs.length} of ${expectedCount} entries`); + } + + return { context, diffs }; +} + +/** + * Extract provider name from the describe block title or the drift report context. + * + * Examples: + * "OpenAI Chat Completions drift" → "OpenAI Chat" + * "OpenAI Chat (non-streaming text)" → "OpenAI Chat" + * "Anthropic Claude drift" → "Anthropic Claude" + */ +function extractProviderName(text: string): string | null { + // Try matching against known provider keys (longest first to avoid partial matches) + const sorted = Object.keys(PROVIDER_MAP).sort((a, b) => b.length - a.length); + for (const key of sorted) { + if (text.includes(key)) return key; + } + return null; +} + +/** + * Extract scenario from the context string. + * + * "OpenAI Chat (non-streaming text)" → "non-streaming text" + * "Anthropic Claude (streaming tool call)" → "streaming tool call" + */ +function extractScenario(context: string): string { + const parenMatch = context.match(/\(([^)]+)\)/); + return parenMatch ? parenMatch[1] : context; +} + +// --------------------------------------------------------------------------- +// Run drift tests and collect results +// --------------------------------------------------------------------------- + +function extractJsonFromString(text: string): VitestJsonResult | null { + const jsonStart = text.indexOf("{"); + const jsonEnd = text.lastIndexOf("}"); + if (jsonStart === -1 || jsonEnd === -1) return null; + try { + const parsed = JSON.parse(text.slice(jsonStart, jsonEnd + 1)) as unknown; + if ( + !parsed || + typeof parsed !== "object" || + !Array.isArray((parsed as Record).testResults) + ) { + console.error( + "extractJsonFromString: parsed JSON does not have testResults array, likely wrong fragment", + ); + return null; + } + return parsed as VitestJsonResult; + } catch (err: unknown) { + console.error( + "extractJsonFromString: failed to parse.", + `Range: [${jsonStart}..${jsonEnd}], length: ${text.length}`, + err instanceof Error ? err.message : String(err), + ); + return null; + } +} + +function hasStdout(err: unknown): err is { stdout: string; stderr?: string } { + return ( + typeof err === "object" && + err !== null && + "stdout" in err && + typeof (err as { stdout: unknown }).stdout === "string" + ); +} + +function parseVitestOutput(stdout: string, context: string): VitestJsonResult | null { + try { + return JSON.parse(stdout) as VitestJsonResult; + } catch (parseErr: unknown) { + console.error( + `${context}:`, + parseErr instanceof Error ? parseErr.message : String(parseErr), + `stdout length: ${stdout.length}`, + ); + return extractJsonFromString(stdout); + } +} + +function runDriftTests(): VitestJsonResult { + try { + const stdout = execSync("pnpm test:drift --reporter=json", { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + maxBuffer: 50 * 1024 * 1024, + }); + const result = parseVitestOutput(stdout, "JSON parse of successful vitest run failed"); + if (result) return result; + throw new Error("Drift tests passed but produced unparseable output"); + } catch (err: unknown) { + // execSync throws on non-zero exit — vitest exits 1 when tests fail + if (hasStdout(err)) { + const result = parseVitestOutput(err.stdout, "Primary JSON parse of vitest stdout failed"); + if (result) return result; + console.error( + "Failed to parse JSON from drift test stdout. Original error:", + err instanceof Error ? err.message : String(err), + ); + if (err.stderr) console.error("stderr:", err.stderr); + } + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`Failed to run drift tests: ${msg}`); + } +} + +function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { + const entries: DriftEntry[] = []; + const unmapped: string[] = []; + let unparseable = 0; + + for (const file of results.testResults) { + for (const assertion of file.assertionResults) { + if (assertion.status !== "failed") continue; + if (assertion.failureMessages.length === 0) continue; + + const fullMessage = assertion.failureMessages.join("\n"); + const parsed = parseDriftBlock(fullMessage); + if (!parsed || parsed.diffs.length === 0) { + unparseable++; + continue; + } + + // Determine provider from ancestor titles (describe block) or context + const ancestorText = assertion.ancestorTitles.join(" "); + const provider = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); + if (!provider) { + unmapped.push(`${ancestorText} > ${assertion.title}`); + continue; + } + + const mapping = PROVIDER_MAP[provider]; + if (!mapping) { + unmapped.push(`${ancestorText} > ${assertion.title} (provider: ${provider})`); + continue; + } + + entries.push({ + provider, + scenario: extractScenario(parsed.context), + builderFile: mapping.builderFile, + builderFunctions: mapping.builderFunctions, + typesFile: mapping.typesFile, + sdkShapesFile: SDK_SHAPES_FILE, + diffs: parsed.diffs, + }); + } + } + + if (unmapped.length > 0) { + console.error(`ERROR: ${unmapped.length} drift failure(s) could not be mapped to a provider:`); + for (const u of unmapped) console.error(` - ${u}`); + throw new Error(`${unmapped.length} unmapped drift entries — update PROVIDER_MAP`); + } + + if (unparseable > 0 && entries.length === 0) { + console.error( + `ERROR: ${unparseable} test failure(s) could not be parsed as drift reports.`, + "This may indicate broken test infrastructure or a changed report format.", + ); + throw new Error(`${unparseable} unparseable test failures with 0 drift entries — investigate`); + } else if (unparseable > 0) { + console.warn( + `WARNING: ${unparseable} test failure(s) did not contain parseable drift data (${entries.length} drift entries collected).`, + ); + } + + return entries; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +function main(): void { + const args = process.argv.slice(2); + const outIndex = args.indexOf("--out"); + const outPath = resolve( + outIndex !== -1 && args[outIndex + 1] ? args[outIndex + 1] : "drift-report.json", + ); + + console.log("Running drift tests..."); + const results = runDriftTests(); + + console.log("Collecting drift entries..."); + const entries = collectDriftEntries(results); + + const report: DriftReport = { + timestamp: new Date().toISOString(), + entries, + }; + + try { + writeFileSync(outPath, JSON.stringify(report, null, 2) + "\n", "utf-8"); + } catch (err) { + console.error(`Failed to write drift report to ${outPath}:`, err); + console.log(JSON.stringify(report, null, 2)); + process.exit(1); + } + console.log(`Drift report written to ${outPath}`); + console.log(` Entries: ${entries.length}`); + + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + console.log(` Critical diffs: ${criticalCount}`); + + if (criticalCount > 0) { + console.log("Exiting with code 2 (critical diffs found)."); + process.exit(2); + } + + console.log("No critical diffs found."); +} + +try { + main(); +} catch (err: unknown) { + console.error("Fatal error:", err); + process.exit(1); +} diff --git a/scripts/drift-types.ts b/scripts/drift-types.ts new file mode 100644 index 00000000..5eaec247 --- /dev/null +++ b/scripts/drift-types.ts @@ -0,0 +1,40 @@ +/** + * Shared types for the drift remediation pipeline. + * + * Used by both drift-report-collector.ts and fix-drift.ts. + */ + +/** + * NOTE: DriftSeverity is intentionally defined in multiple places: + * 1. Here (drift-types.ts) — canonical source, used by the pipeline scripts + * 2. src/__tests__/drift/schema.ts — used by the drift test framework (ShapeDiff) + * 3. src/__tests__/drift-collector.test.ts — local copy for the test helper + * + * Deduplication would require importing across component boundaries. + * If you add a new severity level, update all three locations. + */ +export type DriftSeverity = "critical" | "warning" | "info"; + +export interface ParsedDiff { + path: string; + severity: DriftSeverity; + issue: string; + expected: string; + real: string; + mock: string; +} + +export interface DriftEntry { + provider: string; + scenario: string; + builderFile: string; + builderFunctions: string[]; + typesFile: string | null; + sdkShapesFile: string; + diffs: ParsedDiff[]; +} + +export interface DriftReport { + timestamp: string; + entries: DriftEntry[]; +} diff --git a/scripts/fix-drift.ts b/scripts/fix-drift.ts new file mode 100644 index 00000000..07d12d68 --- /dev/null +++ b/scripts/fix-drift.ts @@ -0,0 +1,681 @@ +/// + +/** + * Drift Fix Orchestrator + * + * Reads a drift-report.json (produced by drift-report-collector.ts), constructs + * a structured prompt, and invokes Claude Code CLI to auto-fix the drift. + * + * Modes: + * Default: npx tsx scripts/fix-drift.ts + * PR mode: npx tsx scripts/fix-drift.ts --create-pr + * Issue mode: npx tsx scripts/fix-drift.ts --create-issue + * + * Exit codes: + * 0 — success (or issue created successfully in --create-issue mode) + * 1 — failure + * 2 — no source files changed (--create-pr mode, nothing to commit) + * 3 — unhandled error (e.g. bad arguments, missing report, git/gh command failure) + * 124 — Claude Code timed out (default mode) + * In default mode, the exit code is passed through from Claude Code. + */ + +import { spawn, execSync, execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync, existsSync, unlinkSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { DriftReport, DriftSeverity } from "./drift-types.js"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** 30-minute hard ceiling for the Claude Code subprocess */ +const CLAUDE_TIMEOUT_MS = 30 * 60 * 1000; + +/** Grace period between SIGTERM and SIGKILL */ +const KILL_GRACE_MS = 10_000; + +const VALID_SEVERITIES: ReadonlySet = new Set(["critical", "warning", "info"]); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +export function todayStamp(): string { + return new Date().toISOString().slice(0, 10); +} + +/** + * Format an exec error into a human-readable Error object. + * Includes exit status, signal, and stderr when available. + * Logs stderr to console.error as a side effect when present. + */ +function formatExecError(cmd: string, err: unknown): Error { + const e = err as { status?: number; signal?: string; stderr?: string | Buffer }; + const detail = [ + e.status !== undefined ? `exit ${e.status}` : null, + e.signal ? `signal ${e.signal}` : null, + e.stderr ? String(e.stderr).trim() : null, + ] + .filter(Boolean) + .join(", "); + const msg = `Command failed: ${cmd}${detail ? ` (${detail})` : ""}`; + if (e.stderr) console.error(msg); + return new Error(msg); +} + +/** + * Run a shell command and return its trimmed stdout. + * + * WARNING: This function passes the command string directly to a shell. + * NEVER call it with interpolated values — use execFileSafe() for commands + * with dynamic arguments. + */ +function exec(cmd: string): string { + try { + return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim(); + } catch (err: unknown) { + throw formatExecError(cmd, err); + } +} + +/** + * Run a command safely without shell interpolation. + * Use this for all commands with dynamic arguments. + */ +export function execFileSafe(file: string, args: string[]): void { + try { + execFileSync(file, args, { stdio: "inherit" }); + } catch (err: unknown) { + throw formatExecError(`${file} ${args.join(" ")}`, err); + } +} + +export function readFileIfExists(path: string): string | null { + if (!existsSync(path)) return null; + return readFileSync(path, "utf-8"); +} + +export function readDriftReport(path: string): DriftReport { + if (!existsSync(path)) { + throw new Error(`Drift report not found at ${path}`); + } + const raw = readFileSync(path, "utf-8"); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err: unknown) { + throw new Error( + `Drift report at ${path} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, + ); + } + if ( + !parsed || + typeof parsed !== "object" || + !Array.isArray((parsed as Record).entries) + ) { + throw new Error(`Drift report at ${path} has invalid structure: expected { entries: [...] }`); + } + if (typeof (parsed as Record).timestamp !== "string") { + throw new Error('Drift report missing "timestamp" field'); + } + const report = parsed as DriftReport; + + // Validate individual entry fields to catch malformed reports early + for (let i = 0; i < report.entries.length; i++) { + const entry = report.entries[i]; + if (!entry || typeof entry.provider !== "string" || !entry.provider) { + throw new Error(`Drift report entry[${i}] missing required "provider" field`); + } + if (!entry.builderFile || typeof entry.builderFile !== "string") { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "builderFile"`); + } + if ( + !Array.isArray(entry.builderFunctions) || + entry.builderFunctions.length === 0 || + !entry.builderFunctions.every((f: unknown) => typeof f === "string") + ) { + throw new Error( + `Drift report entry[${i}] (${entry.provider}) "builderFunctions" must be non-empty string array`, + ); + } + if (!entry.scenario || typeof entry.scenario !== "string") { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "scenario"`); + } + if (!entry.sdkShapesFile || typeof entry.sdkShapesFile !== "string") { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "sdkShapesFile"`); + } + if (entry.typesFile !== null && typeof entry.typesFile !== "string") { + throw new Error( + `Drift report entry[${i}] (${entry.provider}) "typesFile" must be string or null`, + ); + } + if (!Array.isArray(entry.diffs)) { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "diffs" array`); + } + for (let j = 0; j < entry.diffs.length; j++) { + const diff = entry.diffs[j]; + if (!diff.path || typeof diff.path !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "path"`); + } + if (!diff.issue || typeof diff.issue !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "issue"`); + } + if (typeof diff.expected !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "expected"`); + } + if (typeof diff.real !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "real"`); + } + if (typeof diff.mock !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "mock"`); + } + if (!VALID_SEVERITIES.has(diff.severity)) { + throw new Error( + `Drift report entry[${i}].diffs[${j}]: invalid severity "${diff.severity}" — expected one of: ${[...VALID_SEVERITIES].join(", ")}`, + ); + } + } + } + + return report; +} + +// --------------------------------------------------------------------------- +// Prompt construction +// --------------------------------------------------------------------------- + +export function buildPrompt(report: DriftReport): string { + const lines: string[] = []; + + lines.push("You are fixing API drift in the llmock mock server."); + lines.push(""); + lines.push("## Workflow"); + lines.push(""); + lines.push("Follow this exact workflow for each drift fix:"); + lines.push(""); + lines.push("1. RED: Confirm the drift test currently fails by running:"); + lines.push(' pnpm test:drift 2>&1 | grep -A5 "DRIFT"'); + lines.push(""); + lines.push("2. Fix the builder function to add/modify the field matching the real API shape."); + lines.push(" Also fix the corresponding builder for the same provider (e.g., if non-streaming"); + lines.push(" text drifted, also fix non-streaming tool call since they share the same message"); + lines.push(" structure)."); + lines.push(""); + lines.push("3. If the builder file uses TypeScript interfaces from src/types.ts, update those."); + lines.push(""); + lines.push("4. Update the SDK shape in src/__tests__/drift/sdk-shapes.ts if the corresponding"); + lines.push(" shape function doesn't include the new field."); + lines.push(""); + lines.push("5. GREEN: Run pnpm test to verify conformance tests pass."); + lines.push(""); + lines.push("6. Run pnpm test:drift to verify drift is resolved."); + lines.push(""); + lines.push("7. Run npx prettier --write on all changed files."); + lines.push(""); + lines.push("8. REFACTOR: Review your changes for unnecessary complexity."); + lines.push(""); + lines.push("## Drift Entries"); + lines.push(""); + + for (let i = 0; i < report.entries.length; i++) { + const entry = report.entries[i]; + lines.push(`DRIFT ${i + 1}: ${entry.provider} — ${entry.scenario}`); + lines.push(` File: ${entry.builderFile}`); + lines.push(` Functions: ${entry.builderFunctions.join(", ")}`); + lines.push(` Types file: ${entry.typesFile ?? "N/A"}`); + lines.push(` SDK shapes: ${entry.sdkShapesFile}`); + lines.push(" Diffs:"); + for (const diff of entry.diffs) { + lines.push(` - [${diff.severity}] ${diff.issue}`); + lines.push(` Path: ${diff.path}`); + lines.push(` Real API: ${diff.real}`); + lines.push(` Mock: ${diff.mock}`); + } + lines.push(""); + } + + lines.push("## After all fixes"); + lines.push(""); + lines.push("1. Run the full test suite: pnpm test"); + lines.push("2. Run drift verification: pnpm test:drift"); + lines.push("3. Format: npx prettier --write src/ src/__tests__/"); + lines.push("4. Lint: npx eslint src/ src/__tests__/ --fix"); + + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// Claude Code invocation (default mode) +// --------------------------------------------------------------------------- + +function invokeClaudeCode(prompt: string): Promise { + return new Promise((done, reject) => { + const args = [ + "@anthropic-ai/claude-code", + "--print", + "--verbose", + "-p", + prompt, + "--allowedTools", + [ + "Read", + "Edit", + "Write", + "Glob", + "Grep", + "Bash(pnpm test)", + "Bash(pnpm test:drift)", + "Bash(pnpm test:drift *)", + "Bash(npx prettier *)", + "Bash(npx eslint *)", + "Bash(git diff *)", + "Bash(git status *)", + "Bash(git log *)", + ].join(","), + "--max-turns", + "50", + ]; + + const child = spawn("npx", args, { + stdio: ["inherit", "pipe", "pipe"], + }); + + const logChunks: Buffer[] = []; + let killGraceTimer: NodeJS.Timeout | undefined; + let timedOut = false; + + const killTimer = setTimeout(() => { + timedOut = true; + console.error( + `Claude Code timed out after ${CLAUDE_TIMEOUT_MS / 60000} minutes. Sending SIGTERM...`, + ); + child.kill("SIGTERM"); + killGraceTimer = setTimeout(() => { + if (!child.killed) { + console.error("Process did not exit after SIGTERM. Sending SIGKILL..."); + child.kill("SIGKILL"); + } + }, KILL_GRACE_MS); + }, CLAUDE_TIMEOUT_MS); + + child.on("error", (err) => { + clearTimeout(killTimer); + console.error("Failed to spawn Claude Code process:", err.message); + try { + writeFileSync("claude-code-output.log", `Spawn error: ${err.message}\n`, "utf-8"); + } catch (writeErr) { + console.error( + "Failed to write claude-code-output.log:", + writeErr instanceof Error ? writeErr.message : writeErr, + ); + } + reject(err); + }); + + child.stdout.on("data", (chunk: Buffer) => { + process.stdout.write(chunk); + logChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + process.stderr.write(chunk); + logChunks.push(chunk); + }); + + child.on("close", (code, signal) => { + clearTimeout(killTimer); + if (killGraceTimer) clearTimeout(killGraceTimer); + const logContent = Buffer.concat(logChunks).toString("utf-8"); + try { + writeFileSync("claude-code-output.log", logContent, "utf-8"); + } catch (writeErr) { + console.error( + "Failed to write claude-code-output.log:", + writeErr instanceof Error ? writeErr.message : writeErr, + ); + } + if (code === null && signal) { + console.error(`Claude Code process killed by signal: ${signal}`); + } + done(timedOut ? 124 : (code ?? 1)); + }); + }); +} + +// --------------------------------------------------------------------------- +// PR mode (--create-pr) +// --------------------------------------------------------------------------- + +export function patchBumpVersion(): string { + const pkgPath = resolve("package.json"); + const pkg = JSON.parse(readFileSync(pkgPath, "utf-8")) as { + version: string; + [key: string]: unknown; + }; + const parts = pkg.version.split(".").map(Number); + if (parts.length !== 3 || parts.some(isNaN)) { + throw new Error(`Cannot patch-bump non-standard version: ${pkg.version}`); + } + parts[2] += 1; + const newVersion = parts.join("."); + pkg.version = newVersion; + writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n", "utf-8"); + return newVersion; +} + +export function addChangelogEntry(report: DriftReport, version: string): void { + const changelogPath = resolve("CHANGELOG.md"); + const existing = readFileIfExists(changelogPath) ?? ""; + + const providerSummaries = report.entries.map((entry) => { + const fields = entry.diffs.map((d) => d.path).join(", "); + return `- ${entry.provider} (${entry.scenario}): ${fields}`; + }); + + const newEntry = [ + `## ${version}`, + "", + "### Patch Changes", + "", + "- Auto-remediate API drift:", + ...providerSummaries.map((s) => ` ${s}`), + "", + ].join("\n"); + + // Insert after the first line (the title) + const titleLine = "# @copilotkit/llmock\n"; + if (existing.startsWith(titleLine)) { + const rest = existing.slice(titleLine.length); + writeFileSync(changelogPath, titleLine + "\n" + newEntry + rest, "utf-8"); + } else { + writeFileSync(changelogPath, newEntry + "\n" + existing, "utf-8"); + } +} + +export function buildPrBody(report: DriftReport): string { + const providers: string[] = []; + const diffs: string[] = []; + + for (const entry of report.entries) { + providers.push(`- ${entry.provider}: ${entry.scenario}`); + for (const diff of entry.diffs) { + diffs.push(`- \`${diff.path}\`: ${diff.issue}`); + } + } + + const reportJson = JSON.stringify(report, null, 2); + + return [ + "## Summary", + "", + "Auto-generated drift remediation.", + "", + "### Providers affected", + ...providers, + "", + "### Diffs fixed", + ...diffs, + "", + "## Drift Report", + "", + "
", + "Full drift report JSON", + "", + "```json", + reportJson, + "```", + "", + "
", + ].join("\n"); +} + +/** + * Parse a single line from `git status --porcelain` output into a file path. + * Handles quoted paths (special characters) and rename notation (old -> new). + */ +export function parsePorcelainLine(line: string): string { + let path = line.slice(3).trim(); + // Handle renames first: "old -> new" → take the new path + const arrowIdx = path.indexOf(" -> "); + if (arrowIdx !== -1) { + path = path.slice(arrowIdx + 4); + } + // Then strip quotes (git quotes paths with special characters) + if (path.startsWith('"') && path.endsWith('"')) { + path = path.slice(1, -1); + } + return path; +} + +/** + * Return the list of changed files from `git status --porcelain`. + */ +export function getChangedFiles(): string[] { + return exec("git status --porcelain").split("\n").filter(Boolean).map(parsePorcelainLine); +} + +function createPr(report: DriftReport): void { + const stamp = todayStamp(); + + // Determine branch name + let currentBranch: string; + try { + currentBranch = exec("git rev-parse --abbrev-ref HEAD"); + } catch (err: unknown) { + throw new Error(`Cannot determine current branch for PR creation: ${(err as Error).message}`); + } + + const branchName = + currentBranch === "master" || currentBranch === "main" || currentBranch === "HEAD" + ? `fix/drift-${stamp}` + : currentBranch; + + if (branchName !== currentBranch) { + execFileSafe("git", ["checkout", "-b", branchName]); + console.log(`Created branch ${branchName}`); + } + + // Stage and commit in groups — detect uncommitted changes (staged + unstaged) + const changedFiles = getChangedFiles(); + + const builderFiles = changedFiles.filter( + (f) => f.startsWith("src/") && !f.startsWith("src/__tests__/"), + ); + const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/")); + + // Abort if no source files were changed — a version-bump-only PR would be misleading + if (builderFiles.length === 0 && testFiles.length === 0) { + console.error( + "ERROR: No source files changed. Claude Code may not have made any fixes, " + + "or all changes were reverted during verification. Aborting PR creation.", + ); + process.exit(2); + } + + if (builderFiles.length > 0) { + execFileSafe("git", ["add", ...builderFiles]); + execFileSafe("git", ["commit", "-m", "fix: auto-remediate API drift in builder functions"]); + } + + if (testFiles.length > 0) { + execFileSafe("git", ["add", ...testFiles]); + execFileSafe("git", ["commit", "-m", "test: update SDK shapes for drift remediation"]); + } + + const newVersion = patchBumpVersion(); + console.log(`Bumped version to ${newVersion}`); + + addChangelogEntry(report, newVersion); + console.log("Added CHANGELOG.md entry"); + + // Always commit version bump + changelog + execFileSafe("git", ["add", "package.json", "CHANGELOG.md"]); + execFileSafe("git", ["commit", "-m", `chore: bump version to ${newVersion}`, "--allow-empty"]); + + // Catch any remaining files + const remaining = getChangedFiles(); + if (remaining.length > 0) { + execFileSafe("git", ["add", ...remaining]); + execFileSafe("git", ["commit", "-m", "fix: remaining drift remediation changes"]); + } + + execFileSafe("git", ["push", "-u", "origin", branchName]); + console.log(`Pushed branch ${branchName}`); + + const prBody = buildPrBody(report); + const prTitle = `fix: auto-remediate API drift (${stamp})`; + + const prBodyFile = `/tmp/llmock-drift-${process.pid}-pr-body.md`; + writeFileSync(prBodyFile, prBody, "utf-8"); + try { + execFileSafe("gh", [ + "pr", + "create", + "--title", + prTitle, + "--assignee", + "jpr5", + "--body-file", + prBodyFile, + ]); + } finally { + try { + unlinkSync(prBodyFile); + } catch (cleanupErr) { + console.warn( + `Could not clean up temp file:`, + cleanupErr instanceof Error ? cleanupErr.message : cleanupErr, + ); + } + } + + console.log("PR created successfully."); +} + +// --------------------------------------------------------------------------- +// Issue mode (--create-issue) +// --------------------------------------------------------------------------- + +function createIssue(report: DriftReport | null): void { + const stamp = todayStamp(); + const reportJson = report + ? JSON.stringify(report, null, 2) + : "(drift report was not generated — collector may have crashed)"; + const claudeOutput = + readFileIfExists(resolve("claude-code-output.log")) ?? "(no output captured)"; + + const issueBody = [ + "## Drift detected but auto-fix failed", + "", + "The automated drift remediation pipeline detected API drift but was unable", + "to fix it automatically. Manual intervention is required.", + "", + "### Drift Report", + "", + "```json", + reportJson, + "```", + "", + "### Claude Code Output", + "", + "
", + "Full output", + "", + "```", + claudeOutput, + "```", + "", + "
", + ].join("\n"); + + const issueTitle = `Drift detected — auto-fix failed (${stamp})`; + + const issueBodyFile = `/tmp/llmock-drift-${process.pid}-issue-body.md`; + writeFileSync(issueBodyFile, issueBody, "utf-8"); + try { + execFileSafe("gh", [ + "issue", + "create", + "--title", + issueTitle, + "--body-file", + issueBodyFile, + "--label", + "drift", + ]); + } finally { + try { + unlinkSync(issueBodyFile); + } catch (cleanupErr) { + console.warn( + `Could not clean up temp file:`, + cleanupErr instanceof Error ? cleanupErr.message : cleanupErr, + ); + } + } + + console.log("Issue created successfully."); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +export function parseMode(args: string[]): "pr" | "issue" | "default" { + if (args.includes("--create-pr")) return "pr"; + if (args.includes("--create-issue")) return "issue"; + return "default"; +} + +async function main(): Promise { + const args = process.argv.slice(2); + const mode = parseMode(args); + + const reportIndex = args.indexOf("--report"); + const reportPath = resolve( + reportIndex !== -1 && args[reportIndex + 1] ? args[reportIndex + 1] : "drift-report.json", + ); + + // Issue mode handles missing reports gracefully (the safety net shouldn't crash) + if (mode === "issue") { + let report: DriftReport | null = null; + try { + report = readDriftReport(reportPath); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.warn(`Could not read drift report (${msg}), creating issue with available info`); + } + createIssue(report); + return; + } + + const report = readDriftReport(reportPath); + + if (report.entries.length === 0) { + console.log("No drift entries found. Nothing to do."); + process.exit(0); + } + + console.log(`Loaded drift report: ${report.entries.length} entries from ${report.timestamp}`); + + if (mode === "pr") { + createPr(report); + } else { + const prompt = buildPrompt(report); + console.log("Invoking Claude Code CLI..."); + const exitCode = await invokeClaudeCode(prompt); + console.log(`Claude Code exited with code ${exitCode}`); + process.exit(exitCode); + } +} + +const isMain = process.argv[1] === fileURLToPath(import.meta.url); +if (isMain) { + main().catch((err: unknown) => { + console.error("Fatal error:", err); + process.exit(3); + }); +} diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json new file mode 100644 index 00000000..5c934e8d --- /dev/null +++ b/scripts/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["."] +} diff --git a/src/__tests__/cli.test.ts b/src/__tests__/cli.test.ts index 149e2ab7..23558683 100644 --- a/src/__tests__/cli.test.ts +++ b/src/__tests__/cli.test.ts @@ -157,7 +157,7 @@ describe.skipIf(!CLI_AVAILABLE)("CLI: fixture loading", () => { it("fails with error when --fixtures points to a non-existent path", async () => { const { stderr, code } = await runCli(["--fixtures", "/nonexistent/path/to/fixtures"]); - expect(stderr).toContain("Failed to load fixtures"); + expect(stderr).toContain("Fixtures path not found"); expect(code).toBe(1); }); }); diff --git a/src/__tests__/drift-collector.test.ts b/src/__tests__/drift-collector.test.ts new file mode 100644 index 00000000..813f8ea8 --- /dev/null +++ b/src/__tests__/drift-collector.test.ts @@ -0,0 +1,544 @@ +/** + * Tests for key functions in scripts/drift-report-collector.ts + * + * Since scripts/ is outside the rootDir for the main tsconfig (and vitest + * only covers src/__tests__), these functions are duplicated here as local + * test helpers to keep the test runner config intact. Any changes to the + * originals must be reflected here. + */ + +import { describe, it, expect } from "vitest"; +import { formatDriftReport } from "./drift/schema.js"; +import type { ShapeDiff } from "./drift/schema.js"; + +// --------------------------------------------------------------------------- +// Local copies of the types and functions under test +// (mirrors scripts/drift-report-collector.ts — keep in sync) +// --------------------------------------------------------------------------- + +type DriftSeverity = "critical" | "warning" | "info"; + +interface ParsedDiff { + path: string; + severity: DriftSeverity; + issue: string; + expected: string; + real: string; + mock: string; +} + +interface VitestJsonResult { + testResults: VitestTestFile[]; +} + +interface VitestTestFile { + assertionResults: VitestAssertion[]; +} + +interface VitestAssertion { + status: string; + ancestorTitles: string[]; + title: string; + failureMessages: string[]; +} + +interface ProviderMapping { + builderFile: string; + builderFunctions: string[]; + typesFile: string | null; +} + +const PROVIDER_MAP: Record = { + "OpenAI Chat": { + builderFile: "src/helpers.ts", + builderFunctions: [ + "buildTextCompletion", + "buildToolCallCompletion", + "buildTextChunks", + "buildToolCallChunks", + ], + typesFile: "src/types.ts", + }, + "OpenAI Responses": { + builderFile: "src/responses.ts", + builderFunctions: [ + "buildTextResponse", + "buildToolCallResponse", + "buildTextStreamEvents", + "buildToolCallStreamEvents", + ], + typesFile: null, + }, + Anthropic: { + builderFile: "src/messages.ts", + builderFunctions: [ + "buildClaudeTextResponse", + "buildClaudeToolCallResponse", + "buildClaudeTextStreamEvents", + "buildClaudeToolCallStreamEvents", + ], + typesFile: null, + }, + "Anthropic Claude": { + builderFile: "src/messages.ts", + builderFunctions: [ + "buildClaudeTextResponse", + "buildClaudeToolCallResponse", + "buildClaudeTextStreamEvents", + "buildClaudeToolCallStreamEvents", + ], + typesFile: null, + }, + "Google Gemini": { + builderFile: "src/gemini.ts", + builderFunctions: [ + "buildGeminiTextResponse", + "buildGeminiToolCallResponse", + "buildGeminiTextStreamChunks", + "buildGeminiToolCallStreamChunks", + ], + typesFile: null, + }, + Gemini: { + builderFile: "src/gemini.ts", + builderFunctions: [ + "buildGeminiTextResponse", + "buildGeminiToolCallResponse", + "buildGeminiTextStreamChunks", + "buildGeminiToolCallStreamChunks", + ], + typesFile: null, + }, + "OpenAI Realtime": { + builderFile: "src/ws-realtime.ts", + builderFunctions: ["handleWebSocketRealtime", "realtimeItemsToMessages"], + typesFile: null, + }, + "OpenAI Responses WS": { + builderFile: "src/ws-responses.ts", + builderFunctions: ["handleWebSocketResponses"], + typesFile: null, + }, + "Gemini Live": { + builderFile: "src/ws-gemini-live.ts", + builderFunctions: ["handleWebSocketGeminiLive"], + typesFile: null, + }, +}; + +const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; + +const VALID_SEVERITIES = new Set(["critical", "warning", "info"]); + +function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { + const headerMatch = text.match(/API DRIFT DETECTED:\s*(.+)/); + if (!headerMatch) return null; + + const context = headerMatch[1].trim(); + const diffs: ParsedDiff[] = []; + + const entryPattern = + /\d+\.\s*\[(\w+)\]\s*(.+)\n\s*Path:\s*(.+)\n\s*SDK:\s*(.+)\n\s*Real:\s*(.+)\n\s*Mock:\s*(.+)/g; + + let match: RegExpExecArray | null; + while ((match = entryPattern.exec(text)) !== null) { + const severity = match[1].trim(); + if (!VALID_SEVERITIES.has(severity as DriftSeverity)) continue; + diffs.push({ + severity: severity as DriftSeverity, + issue: match[2].trim(), + path: match[3].trim(), + expected: match[4].trim(), + real: match[5].trim(), + mock: match[6].trim(), + }); + } + + return { context, diffs }; +} + +function extractProviderName(text: string): string | null { + const sorted = Object.keys(PROVIDER_MAP).sort((a, b) => b.length - a.length); + for (const key of sorted) { + if (text.includes(key)) return key; + } + return null; +} + +function extractScenario(context: string): string { + const parenMatch = context.match(/\(([^)]+)\)/); + return parenMatch ? parenMatch[1] : context; +} + +function collectDriftEntries(results: VitestJsonResult): Array<{ + provider: string; + scenario: string; + builderFile: string; + builderFunctions: string[]; + typesFile: string | null; + sdkShapesFile: string; + diffs: ParsedDiff[]; +}> { + const entries: Array<{ + provider: string; + scenario: string; + builderFile: string; + builderFunctions: string[]; + typesFile: string | null; + sdkShapesFile: string; + diffs: ParsedDiff[]; + }> = []; + const unmapped: string[] = []; + let unparseable = 0; + + for (const file of results.testResults) { + for (const assertion of file.assertionResults) { + if (assertion.status !== "failed") continue; + if (assertion.failureMessages.length === 0) continue; + + const fullMessage = assertion.failureMessages.join("\n"); + const parsed = parseDriftBlock(fullMessage); + if (!parsed || parsed.diffs.length === 0) { + unparseable++; + continue; + } + + const ancestorText = assertion.ancestorTitles.join(" "); + const provider = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); + if (!provider) { + unmapped.push(`${ancestorText} > ${assertion.title}`); + continue; + } + + const mapping = PROVIDER_MAP[provider]; + if (!mapping) { + unmapped.push(`${ancestorText} > ${assertion.title} (provider: ${provider})`); + continue; + } + + entries.push({ + provider, + scenario: extractScenario(parsed.context), + builderFile: mapping.builderFile, + builderFunctions: mapping.builderFunctions, + typesFile: mapping.typesFile, + sdkShapesFile: SDK_SHAPES_FILE, + diffs: parsed.diffs, + }); + } + } + + if (unmapped.length > 0) { + throw new Error(`${unmapped.length} unmapped drift entries — update PROVIDER_MAP`); + } + + if (unparseable > 0 && entries.length === 0) { + throw new Error(`${unparseable} unparseable test failures with 0 drift entries — investigate`); + } + + return entries; +} + +// --------------------------------------------------------------------------- +// Helpers for building test fixtures +// --------------------------------------------------------------------------- + +function makeResult(assertions: VitestAssertion[]): VitestJsonResult { + return { testResults: [{ assertionResults: assertions }] }; +} + +function makeAssertion(overrides: Partial = {}): VitestAssertion { + return { + status: "failed", + ancestorTitles: [], + title: "test title", + failureMessages: [], + ...overrides, + }; +} + +const SAMPLE_DIFF: ShapeDiff = { + path: "choices[0].message.refusal", + severity: "critical", + issue: "LLMOCK DRIFT — field in SDK + real API but missing from mock", + expected: "null", + real: "null", + mock: "", +}; + +const SAMPLE_DIFF_WARNING: ShapeDiff = { + path: "choices[0].message.extra", + severity: "warning", + issue: "PROVIDER ADDED FIELD — in real API but not in SDK or mock", + expected: "", + real: "string", + mock: "", +}; + +// --------------------------------------------------------------------------- +// parseDriftBlock tests +// --------------------------------------------------------------------------- + +describe("parseDriftBlock", () => { + it("returns null for text with no API DRIFT DETECTED header", () => { + expect(parseDriftBlock("")).toBeNull(); + expect(parseDriftBlock("Error: AssertionError: expected true to be false")).toBeNull(); + expect(parseDriftBlock("No drift detected: OpenAI Chat (non-streaming text)")).toBeNull(); + }); + + it("parses a single drift entry correctly", () => { + const formatted = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = parseDriftBlock(formatted); + + expect(result).not.toBeNull(); + expect(result!.context).toBe("OpenAI Chat (non-streaming text)"); + expect(result!.diffs).toHaveLength(1); + + const diff = result!.diffs[0]; + expect(diff.severity).toBe("critical"); + expect(diff.path).toBe("choices[0].message.refusal"); + expect(diff.issue).toBe("LLMOCK DRIFT — field in SDK + real API but missing from mock"); + expect(diff.expected).toBe("null"); + expect(diff.real).toBe("null"); + expect(diff.mock).toBe(""); + }); + + it("parses multiple drift entries", () => { + const formatted = formatDriftReport("OpenAI Chat (non-streaming text)", [ + SAMPLE_DIFF, + SAMPLE_DIFF_WARNING, + ]); + const result = parseDriftBlock(formatted); + + expect(result).not.toBeNull(); + expect(result!.diffs).toHaveLength(2); + expect(result!.diffs[0].severity).toBe("critical"); + expect(result!.diffs[1].severity).toBe("warning"); + expect(result!.diffs[1].path).toBe("choices[0].message.extra"); + }); + + it("skips entries with unknown severity", () => { + // Manually construct a report with a bad severity + const text = ` +API DRIFT DETECTED: OpenAI Chat (test) + + 1. [unknown] Some issue + Path: foo.bar + SDK: string + Real: string + Mock: + + 2. [critical] Real issue + Path: baz.qux + SDK: null + Real: null + Mock: +`; + const result = parseDriftBlock(text); + expect(result).not.toBeNull(); + // Only the critical entry should be in diffs + expect(result!.diffs).toHaveLength(1); + expect(result!.diffs[0].severity).toBe("critical"); + expect(result!.diffs[0].path).toBe("baz.qux"); + }); + + it("handles context strings with parenthetical scenario", () => { + const formatted = formatDriftReport("Anthropic Claude (streaming tool call)", [SAMPLE_DIFF]); + const result = parseDriftBlock(formatted); + + expect(result).not.toBeNull(); + expect(result!.context).toBe("Anthropic Claude (streaming tool call)"); + }); + + it("round-trips through formatDriftReport for all severity levels", () => { + const diffs: ShapeDiff[] = [ + { ...SAMPLE_DIFF, severity: "critical" }, + { ...SAMPLE_DIFF_WARNING, severity: "warning" }, + { + path: "model", + severity: "info", + issue: "SDK EXTRA — field in SDK but not in real API response", + expected: "string", + real: "", + mock: "string", + }, + ]; + const formatted = formatDriftReport("Google Gemini (non-streaming text)", diffs); + const result = parseDriftBlock(formatted); + + expect(result).not.toBeNull(); + expect(result!.context).toBe("Google Gemini (non-streaming text)"); + expect(result!.diffs).toHaveLength(3); + + for (let i = 0; i < diffs.length; i++) { + expect(result!.diffs[i].severity).toBe(diffs[i].severity); + expect(result!.diffs[i].path).toBe(diffs[i].path); + expect(result!.diffs[i].issue).toBe(diffs[i].issue); + expect(result!.diffs[i].expected).toBe(diffs[i].expected); + expect(result!.diffs[i].real).toBe(diffs[i].real); + expect(result!.diffs[i].mock).toBe(diffs[i].mock); + } + }); +}); + +// --------------------------------------------------------------------------- +// extractProviderName tests +// --------------------------------------------------------------------------- + +describe("extractProviderName", () => { + it("matches exact provider names", () => { + expect(extractProviderName("OpenAI Chat")).toBe("OpenAI Chat"); + expect(extractProviderName("Gemini")).toBe("Gemini"); + expect(extractProviderName("OpenAI Realtime")).toBe("OpenAI Realtime"); + }); + + it("uses longest match — Anthropic Claude over Anthropic", () => { + // "Anthropic Claude" is longer and should win over "Anthropic" + expect(extractProviderName("Anthropic Claude drift")).toBe("Anthropic Claude"); + expect(extractProviderName("Anthropic Claude (streaming tool call)")).toBe("Anthropic Claude"); + }); + + it("uses longest match — Google Gemini over Gemini", () => { + expect(extractProviderName("Google Gemini drift")).toBe("Google Gemini"); + expect(extractProviderName("Google Gemini (non-streaming text)")).toBe("Google Gemini"); + }); + + it("returns null for unknown provider", () => { + expect(extractProviderName("")).toBeNull(); + expect(extractProviderName("Unknown Provider drift")).toBeNull(); + expect(extractProviderName("Cohere drift")).toBeNull(); + }); + + it("matches provider in drift test describe block format", () => { + expect(extractProviderName("OpenAI Chat Completions drift")).toBe("OpenAI Chat"); + expect(extractProviderName("OpenAI Responses API drift")).toBe("OpenAI Responses"); + expect(extractProviderName("Gemini Live WebSocket drift")).toBe("Gemini Live"); + }); + + it("matches provider from context string (parenthetical format)", () => { + expect(extractProviderName("OpenAI Chat (non-streaming text)")).toBe("OpenAI Chat"); + expect(extractProviderName("Anthropic (streaming text)")).toBe("Anthropic"); + }); +}); + +// --------------------------------------------------------------------------- +// collectDriftEntries tests +// --------------------------------------------------------------------------- + +describe("collectDriftEntries", () => { + it("returns empty array when no failed tests", () => { + const result = makeResult([ + makeAssertion({ status: "passed" }), + makeAssertion({ status: "pending" }), + ]); + expect(collectDriftEntries(result)).toEqual([]); + }); + + it("returns empty array when there are no test files at all", () => { + expect(collectDriftEntries({ testResults: [] })).toEqual([]); + }); + + it("throws when an unmapped provider is found in drift report", () => { + const driftText = formatDriftReport("UnknownProvider (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["UnknownProvider drift"], + failureMessages: [driftText], + }), + ]); + expect(() => collectDriftEntries(result)).toThrow(/unmapped drift entries/); + }); + + it("throws when all failures are unparseable and no drift entries collected", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + failureMessages: ["Error: expected true to equal false\n at Object."], + }), + makeAssertion({ + status: "failed", + failureMessages: ["TypeError: Cannot read property 'foo' of undefined"], + }), + ]); + expect(() => collectDriftEntries(result)).toThrow(/unparseable test failures/); + }); + + it("returns valid entries and tolerates unparseable failures mixed in", () => { + const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [driftText], + }), + makeAssertion({ + status: "failed", + ancestorTitles: ["unrelated suite"], + title: "some other failure", + failureMessages: ["Error: plain error with no drift header"], + }), + ]); + + const entries = collectDriftEntries(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Chat"); + expect(entries[0].scenario).toBe("non-streaming text"); + expect(entries[0].builderFile).toBe("src/helpers.ts"); + expect(entries[0].diffs).toHaveLength(1); + expect(entries[0].diffs[0].severity).toBe("critical"); + }); + + it("ignores passed assertions in a mixed result set", () => { + const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ status: "passed", failureMessages: [] }), + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [driftText], + }), + ]); + + const entries = collectDriftEntries(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Chat"); + }); + + it("collects entries from multiple test files", () => { + const openAiDrift = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const geminiDrift = formatDriftReport("Google Gemini (non-streaming text)", [ + SAMPLE_DIFF_WARNING, + ]); + + const results: VitestJsonResult = { + testResults: [ + { + assertionResults: [ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + failureMessages: [openAiDrift], + }), + ], + }, + { + assertionResults: [ + makeAssertion({ + status: "failed", + ancestorTitles: ["Google Gemini drift"], + failureMessages: [geminiDrift], + }), + ], + }, + ], + }; + + const entries = collectDriftEntries(results); + expect(entries).toHaveLength(2); + expect(entries[0].provider).toBe("OpenAI Chat"); + expect(entries[1].provider).toBe("Google Gemini"); + }); +}); diff --git a/src/__tests__/fix-drift.test.ts b/src/__tests__/fix-drift.test.ts new file mode 100644 index 00000000..4927a2df --- /dev/null +++ b/src/__tests__/fix-drift.test.ts @@ -0,0 +1,745 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { resolve } from "node:path"; + +import type { + DriftReport, + DriftEntry, + DriftSeverity, + ParsedDiff, +} from "../../scripts/drift-types.js"; + +// We mock fs and child_process before importing the module under test +vi.mock("node:fs", async () => { + const actual = await vi.importActual("node:fs"); + return { + ...actual, + readFileSync: vi.fn(actual.readFileSync), + writeFileSync: vi.fn(), + existsSync: vi.fn(actual.existsSync), + }; +}); + +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { + ...actual, + execFileSync: vi.fn(), + execSync: vi.fn(), + }; +}); + +import { + todayStamp, + readDriftReport, + buildPrompt, + patchBumpVersion, + addChangelogEntry, + buildPrBody, + parsePorcelainLine, + readFileIfExists, + execFileSafe, + parseMode, + getChangedFiles, +} from "../../scripts/fix-drift.js"; + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { execFileSync, execSync } from "node:child_process"; + +const mockedReadFileSync = vi.mocked(readFileSync); +const mockedWriteFileSync = vi.mocked(writeFileSync); +const mockedExistsSync = vi.mocked(existsSync); +const mockedExecFileSync = vi.mocked(execFileSync); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeDiff(overrides: Partial = {}): ParsedDiff { + return { + path: "response.choices[0].message.content", + severity: "warning", + issue: "missing field", + expected: "string", + real: '"hello"', + mock: "undefined", + ...overrides, + }; +} + +function makeEntry(overrides: Partial = {}): DriftEntry { + return { + provider: "openai", + scenario: "non-streaming text", + builderFile: "src/builders/openai.ts", + builderFunctions: ["buildTextResponse"], + typesFile: "src/types.ts", + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [makeDiff()], + ...overrides, + }; +} + +function makeReport(overrides: Partial = {}): DriftReport { + return { + timestamp: "2026-03-19T00:00:00.000Z", + entries: [makeEntry()], + ...overrides, + }; +} + +// --------------------------------------------------------------------------- +// todayStamp +// --------------------------------------------------------------------------- + +describe("todayStamp", () => { + it("returns a YYYY-MM-DD formatted string", () => { + const result = todayStamp(); + expect(result).toMatch(/^\d{4}-\d{2}-\d{2}$/); + }); + + it("matches today's date", () => { + const expected = new Date().toISOString().slice(0, 10); + expect(todayStamp()).toBe(expected); + }); +}); + +// --------------------------------------------------------------------------- +// readDriftReport +// --------------------------------------------------------------------------- + +describe("readDriftReport", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("parses a valid report", () => { + const report = makeReport(); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + const result = readDriftReport("/tmp/report.json"); + expect(result).toEqual(report); + }); + + it("throws when file does not exist", () => { + mockedExistsSync.mockReturnValue(false); + expect(() => readDriftReport("/tmp/missing.json")).toThrow("Drift report not found"); + }); + + it("throws on invalid JSON", () => { + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue("not json {{{"); + expect(() => readDriftReport("/tmp/bad.json")).toThrow("not valid JSON"); + }); + + it("throws when entries array is missing", () => { + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify({ timestamp: "2026-01-01" })); + expect(() => readDriftReport("/tmp/no-entries.json")).toThrow("invalid structure"); + }); + + it("throws when entries is not an array", () => { + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify({ entries: "not-an-array" })); + expect(() => readDriftReport("/tmp/bad-entries.json")).toThrow("invalid structure"); + }); + + it("throws when timestamp is missing", () => { + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify({ entries: [] })); + expect(() => readDriftReport("/tmp/no-timestamp.json")).toThrow('missing "timestamp"'); + }); + + it("throws when timestamp is not a string", () => { + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify({ entries: [], timestamp: 12345 })); + expect(() => readDriftReport("/tmp/bad-timestamp.json")).toThrow('missing "timestamp"'); + }); + + it("throws when entry is missing provider", () => { + const report = makeReport(); + (report.entries[0] as Record).provider = ""; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-provider.json")).toThrow( + 'entry[0] missing required "provider"', + ); + }); + + it("throws when entry has no diffs array", () => { + const report = makeReport(); + (report.entries[0] as Record).diffs = "not-array"; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-diffs.json")).toThrow('missing "diffs" array'); + }); + + it("throws when a diff has invalid severity", () => { + const report = makeReport({ + entries: [ + makeEntry({ + diffs: [makeDiff({ severity: "extreme" as DriftSeverity })], + }), + ], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/bad-severity.json")).toThrow('invalid severity "extreme"'); + }); + + it("throws when entry is missing builderFile", () => { + const report = makeReport(); + (report.entries[0] as Record).builderFile = ""; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-builder.json")).toThrow('missing "builderFile"'); + }); + + it("throws when entry has empty builderFunctions", () => { + const report = makeReport(); + report.entries[0].builderFunctions = []; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/empty-funcs.json")).toThrow( + '"builderFunctions" must be non-empty string array', + ); + }); + + it("throws when builderFunctions contains non-string elements", () => { + const report = makeReport(); + (report.entries[0] as Record).builderFunctions = ["valid", 42]; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/bad-funcs.json")).toThrow( + '"builderFunctions" must be non-empty string array', + ); + }); + + it("throws when entry is missing scenario", () => { + const report = makeReport(); + (report.entries[0] as Record).scenario = 123; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-scenario.json")).toThrow('missing "scenario"'); + }); + + it("throws when entry is missing sdkShapesFile", () => { + const report = makeReport(); + (report.entries[0] as Record).sdkShapesFile = ""; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-shapes.json")).toThrow('missing "sdkShapesFile"'); + }); + + it("throws when typesFile is not a string or null", () => { + const report = makeReport(); + (report.entries[0] as Record).typesFile = 42; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/bad-types.json")).toThrow( + '"typesFile" must be string or null', + ); + }); + + it("accepts typesFile as null", () => { + const report = makeReport({ entries: [makeEntry({ typesFile: null })] }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/null-types.json")).not.toThrow(); + }); + + it("throws when a diff is missing path", () => { + const report = makeReport({ + entries: [makeEntry({ diffs: [makeDiff({ path: "" })] })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-path.json")).toThrow('missing "path"'); + }); + + it("throws when a diff is missing issue", () => { + const report = makeReport({ + entries: [makeEntry({ diffs: [makeDiff({ issue: "" })] })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-issue.json")).toThrow('missing "issue"'); + }); + + it("throws when a diff is missing expected", () => { + const report = makeReport({ + entries: [makeEntry({ diffs: [makeDiff({ expected: undefined as unknown as string })] })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-expected.json")).toThrow('missing "expected"'); + }); + + it("throws when a diff is missing real", () => { + const report = makeReport({ + entries: [makeEntry({ diffs: [makeDiff({ real: undefined as unknown as string })] })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-real.json")).toThrow('missing "real"'); + }); + + it("throws when a diff is missing mock", () => { + const report = makeReport({ + entries: [makeEntry({ diffs: [makeDiff({ mock: undefined as unknown as string })] })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/no-mock.json")).toThrow('missing "mock"'); + }); + + it("accepts all valid severities", () => { + for (const severity of ["critical", "warning", "info"] as const) { + const report = makeReport({ + entries: [makeEntry({ diffs: [makeDiff({ severity })] })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/ok.json")).not.toThrow(); + } + }); + + it("validates all entries, not just the first", () => { + const report = makeReport({ + entries: [makeEntry({ provider: "openai" }), makeEntry({ provider: "" })], + }); + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(JSON.stringify(report)); + + expect(() => readDriftReport("/tmp/second-bad.json")).toThrow( + 'entry[1] missing required "provider"', + ); + }); +}); + +// --------------------------------------------------------------------------- +// buildPrompt +// --------------------------------------------------------------------------- + +describe("buildPrompt", () => { + it("includes workflow instructions", () => { + const prompt = buildPrompt(makeReport()); + expect(prompt).toContain("## Workflow"); + expect(prompt).toContain("RED:"); + expect(prompt).toContain("GREEN:"); + expect(prompt).toContain("REFACTOR:"); + }); + + it("renders a single drift entry", () => { + const report = makeReport(); + const prompt = buildPrompt(report); + + expect(prompt).toContain("DRIFT 1: openai"); + expect(prompt).toContain("non-streaming text"); + expect(prompt).toContain("File: src/builders/openai.ts"); + expect(prompt).toContain("Functions: buildTextResponse"); + expect(prompt).toContain("Types file: src/types.ts"); + expect(prompt).toContain("[warning] missing field"); + }); + + it("renders multiple drift entries with sequential numbering", () => { + const report = makeReport({ + entries: [ + makeEntry({ provider: "openai", scenario: "streaming" }), + makeEntry({ provider: "anthropic", scenario: "non-streaming" }), + ], + }); + const prompt = buildPrompt(report); + + expect(prompt).toContain("DRIFT 1: openai"); + expect(prompt).toContain("DRIFT 2: anthropic"); + }); + + it('renders "N/A" when typesFile is null', () => { + const report = makeReport({ + entries: [makeEntry({ typesFile: null })], + }); + const prompt = buildPrompt(report); + expect(prompt).toContain("Types file: N/A"); + }); + + it("includes after-fixes section", () => { + const prompt = buildPrompt(makeReport()); + expect(prompt).toContain("## After all fixes"); + expect(prompt).toContain("pnpm test"); + expect(prompt).toContain("pnpm test:drift"); + }); + + it("renders diff details (path, real, mock)", () => { + const diff = makeDiff({ + path: "body.model", + real: '"gpt-4o"', + mock: '"gpt-4"', + }); + const report = makeReport({ entries: [makeEntry({ diffs: [diff] })] }); + const prompt = buildPrompt(report); + + expect(prompt).toContain("Path: body.model"); + expect(prompt).toContain('Real API: "gpt-4o"'); + expect(prompt).toContain('Mock: "gpt-4"'); + }); +}); + +// --------------------------------------------------------------------------- +// patchBumpVersion +// --------------------------------------------------------------------------- + +describe("patchBumpVersion", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('bumps patch version from "1.2.3" to "1.2.4"', () => { + const pkg = { name: "@copilotkit/llmock", version: "1.2.3" }; + mockedReadFileSync.mockReturnValue(JSON.stringify(pkg)); + mockedWriteFileSync.mockImplementation(() => {}); + + const result = patchBumpVersion(); + + expect(result).toBe("1.2.4"); + expect(mockedWriteFileSync).toHaveBeenCalledOnce(); + const written = vi.mocked(writeFileSync).mock.calls[0][1] as string; + expect(JSON.parse(written.trim()).version).toBe("1.2.4"); + }); + + it('bumps "0.0.0" to "0.0.1"', () => { + const pkg = { version: "0.0.0" }; + mockedReadFileSync.mockReturnValue(JSON.stringify(pkg)); + mockedWriteFileSync.mockImplementation(() => {}); + + expect(patchBumpVersion()).toBe("0.0.1"); + }); + + it("throws on non-standard version string", () => { + const pkg = { version: "1.2.3-beta.1" }; + mockedReadFileSync.mockReturnValue(JSON.stringify(pkg)); + + expect(() => patchBumpVersion()).toThrow("non-standard version"); + }); + + it("throws on version with wrong number of parts", () => { + const pkg = { version: "1.2" }; + mockedReadFileSync.mockReturnValue(JSON.stringify(pkg)); + + expect(() => patchBumpVersion()).toThrow("non-standard version"); + }); + + it("writes to the correct path (resolve('package.json'))", () => { + const pkg = { version: "1.0.0" }; + mockedReadFileSync.mockReturnValue(JSON.stringify(pkg)); + mockedWriteFileSync.mockImplementation(() => {}); + + patchBumpVersion(); + + const writtenPath = vi.mocked(writeFileSync).mock.calls[0][0] as string; + expect(writtenPath).toBe(resolve("package.json")); + }); + + it("preserves other fields in package.json", () => { + const pkg = { name: "test-pkg", version: "2.0.0", license: "MIT" }; + mockedReadFileSync.mockReturnValue(JSON.stringify(pkg)); + mockedWriteFileSync.mockImplementation(() => {}); + + patchBumpVersion(); + + const written = JSON.parse((vi.mocked(writeFileSync).mock.calls[0][1] as string).trim()); + expect(written.name).toBe("test-pkg"); + expect(written.license).toBe("MIT"); + }); +}); + +// --------------------------------------------------------------------------- +// addChangelogEntry +// --------------------------------------------------------------------------- + +describe("addChangelogEntry", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("inserts entry after title line when changelog has title", () => { + const existing = "# @copilotkit/llmock\n\n## 1.0.0\n\nOld entry\n"; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(existing); + mockedWriteFileSync.mockImplementation(() => {}); + + const report = makeReport(); + addChangelogEntry(report, "1.0.1"); + + expect(mockedWriteFileSync).toHaveBeenCalledOnce(); + const written = vi.mocked(writeFileSync).mock.calls[0][1] as string; + + // Title is preserved at the top + expect(written.startsWith("# @copilotkit/llmock\n")).toBe(true); + // New version entry comes before old + expect(written.indexOf("## 1.0.1")).toBeLessThan(written.indexOf("## 1.0.0")); + // Contains patch changes section + expect(written).toContain("### Patch Changes"); + expect(written).toContain("Auto-remediate API drift"); + // Contains provider summary + expect(written).toContain("openai (non-streaming text)"); + }); + + it("prepends entry when changelog has no title", () => { + const existing = "## 1.0.0\n\nOld stuff\n"; + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue(existing); + mockedWriteFileSync.mockImplementation(() => {}); + + addChangelogEntry(makeReport(), "1.0.1"); + + const written = vi.mocked(writeFileSync).mock.calls[0][1] as string; + expect(written.startsWith("## 1.0.1")).toBe(true); + expect(written).toContain("## 1.0.0"); + }); + + it("handles empty/missing changelog", () => { + mockedExistsSync.mockReturnValue(false); + mockedReadFileSync.mockImplementation(() => { + throw new Error("ENOENT"); + }); + mockedWriteFileSync.mockImplementation(() => {}); + + // readFileIfExists returns null when !existsSync, so it won't call readFileSync + addChangelogEntry(makeReport(), "0.0.1"); + + const written = vi.mocked(writeFileSync).mock.calls[0][1] as string; + expect(written).toContain("## 0.0.1"); + expect(written).toContain("### Patch Changes"); + }); + + it("includes diff paths in provider summary", () => { + const report = makeReport({ + entries: [ + makeEntry({ + diffs: [makeDiff({ path: "a.b" }), makeDiff({ path: "c.d" })], + }), + ], + }); + mockedExistsSync.mockReturnValue(false); + mockedWriteFileSync.mockImplementation(() => {}); + + addChangelogEntry(report, "1.0.0"); + + const written = vi.mocked(writeFileSync).mock.calls[0][1] as string; + expect(written).toContain("a.b, c.d"); + }); +}); + +// --------------------------------------------------------------------------- +// buildPrBody +// --------------------------------------------------------------------------- + +describe("buildPrBody", () => { + it("contains Summary heading", () => { + const body = buildPrBody(makeReport()); + expect(body).toContain("## Summary"); + expect(body).toContain("Auto-generated drift remediation"); + }); + + it("lists providers affected", () => { + const report = makeReport({ + entries: [ + makeEntry({ provider: "openai", scenario: "streaming" }), + makeEntry({ provider: "anthropic", scenario: "non-streaming" }), + ], + }); + const body = buildPrBody(report); + + expect(body).toContain("### Providers affected"); + expect(body).toContain("- openai: streaming"); + expect(body).toContain("- anthropic: non-streaming"); + }); + + it("lists diffs fixed with code-formatted paths", () => { + const diff = makeDiff({ path: "response.id", issue: "field missing" }); + const report = makeReport({ entries: [makeEntry({ diffs: [diff] })] }); + const body = buildPrBody(report); + + expect(body).toContain("### Diffs fixed"); + expect(body).toContain("- `response.id`: field missing"); + }); + + it("includes a collapsible JSON details block", () => { + const report = makeReport(); + const body = buildPrBody(report); + + expect(body).toContain("
"); + expect(body).toContain("Full drift report JSON"); + expect(body).toContain("```json"); + expect(body).toContain("```"); + expect(body).toContain("
"); + }); + + it("contains the full report JSON", () => { + const report = makeReport(); + const body = buildPrBody(report); + const expectedJson = JSON.stringify(report, null, 2); + expect(body).toContain(expectedJson); + }); +}); + +// --------------------------------------------------------------------------- +// parsePorcelainLine +// --------------------------------------------------------------------------- + +describe("parsePorcelainLine", () => { + it("parses a normal modified file", () => { + expect(parsePorcelainLine(" M src/foo.ts")).toBe("src/foo.ts"); + }); + + it("parses an added file", () => { + expect(parsePorcelainLine("A src/new.ts")).toBe("src/new.ts"); + }); + + it("parses an untracked file", () => { + expect(parsePorcelainLine("?? src/unknown.ts")).toBe("src/unknown.ts"); + }); + + it("handles quoted paths", () => { + expect(parsePorcelainLine(' M "src/special chars.ts"')).toBe("src/special chars.ts"); + }); + + it("handles rename notation, returning the new path", () => { + expect(parsePorcelainLine("R old.ts -> new.ts")).toBe("new.ts"); + }); + + it("handles rename with quoted paths", () => { + expect(parsePorcelainLine('R "old name.ts" -> "new name.ts"')).toBe("new name.ts"); + }); + + it("handles paths with leading/trailing whitespace in the path portion", () => { + // The trim() in parsePorcelainLine handles extra whitespace + expect(parsePorcelainLine("MM src/bar.ts ")).toBe("src/bar.ts"); + }); +}); + +// --------------------------------------------------------------------------- +// readFileIfExists +// --------------------------------------------------------------------------- + +describe("readFileIfExists", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("returns file content when file exists", () => { + mockedExistsSync.mockReturnValue(true); + mockedReadFileSync.mockReturnValue("file content here"); + + expect(readFileIfExists("/tmp/exists.txt")).toBe("file content here"); + }); + + it("returns null when file does not exist", () => { + mockedExistsSync.mockReturnValue(false); + + expect(readFileIfExists("/tmp/missing.txt")).toBeNull(); + expect(mockedReadFileSync).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// execFileSafe +// --------------------------------------------------------------------------- + +describe("execFileSafe", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("calls execFileSync with the correct arguments", () => { + mockedExecFileSync.mockReturnValue(Buffer.from("")); + + execFileSafe("git", ["status"]); + + expect(mockedExecFileSync).toHaveBeenCalledWith("git", ["status"], { stdio: "inherit" }); + }); + + it("throws a formatted error on failure", () => { + const err = Object.assign(new Error("fail"), { status: 128, stderr: "fatal: not a repo" }); + mockedExecFileSync.mockImplementation(() => { + throw err; + }); + + expect(() => execFileSafe("git", ["status"])).toThrow("Command failed: git status"); + }); +}); + +// --------------------------------------------------------------------------- +// parseMode +// --------------------------------------------------------------------------- + +describe("parseMode", () => { + it("returns 'pr' for --create-pr flag", () => { + expect(parseMode(["--create-pr"])).toBe("pr"); + }); + + it("returns 'issue' for --create-issue flag", () => { + expect(parseMode(["--create-issue"])).toBe("issue"); + }); + + it("returns 'default' with no flags", () => { + expect(parseMode([])).toBe("default"); + }); + + it("returns 'default' with unrelated flags", () => { + expect(parseMode(["--report", "drift-report.json"])).toBe("default"); + }); + + it("returns 'pr' even with other flags present", () => { + expect(parseMode(["--report", "drift-report.json", "--create-pr"])).toBe("pr"); + }); +}); + +// --------------------------------------------------------------------------- +// getChangedFiles +// --------------------------------------------------------------------------- + +describe("getChangedFiles", () => { + const mockedExecSync = vi.mocked(execSync); + + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("returns parsed file paths from git status output", () => { + // Note: exec() trims the result, so we use staged-file format (M not M) + // which doesn't have a leading space that trim would strip + mockedExecSync.mockReturnValue("M src/helpers.ts\nM src/server.ts"); + const result = getChangedFiles(); + expect(result).toEqual(["src/helpers.ts", "src/server.ts"]); + }); + + it("returns empty array for empty git status", () => { + mockedExecSync.mockReturnValue(""); + const result = getChangedFiles(); + expect(result).toEqual([]); + }); + + it("handles renamed files", () => { + mockedExecSync.mockReturnValue("R old.ts -> new.ts\n M src/foo.ts\n"); + const result = getChangedFiles(); + expect(result).toEqual(["new.ts", "src/foo.ts"]); + }); +}); diff --git a/src/cli.ts b/src/cli.ts index b7dd2333..2236b6bf 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -87,11 +87,19 @@ async function main() { fixtures = loadFixtureFile(fixturePath, logger); } } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`Failed to load fixtures from ${fixturePath}: ${msg}`); + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + console.error(`Fixtures path not found: ${fixturePath}`); + } else { + const msg = err instanceof Error ? err.message : String(err); + console.error(`Failed to load fixtures from ${fixturePath}: ${msg}`); + } process.exit(1); } + if (fixtures.length === 0) { + console.warn("Warning: No fixtures loaded. The server will return 404 for all requests."); + } + logger.info(`Loaded ${fixtures.length} fixture(s) from ${fixturePath}`); // Validate fixtures if requested diff --git a/src/messages.ts b/src/messages.ts index 0879a120..95f6f180 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -279,6 +279,9 @@ function buildClaudeToolCallStreamEvents( try { argsObj = JSON.parse(tc.arguments || "{}"); } catch { + console.warn( + `[llmock] Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, + ); argsObj = {}; } const argsJson = JSON.stringify(argsObj); @@ -350,6 +353,9 @@ function buildClaudeToolCallResponse(toolCalls: ToolCall[], model: string): obje try { argsObj = JSON.parse(tc.arguments || "{}"); } catch { + console.warn( + `[llmock] Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, + ); argsObj = {}; } return {