diff --git a/evals/suites/generation-quality/cases.jsonl b/evals/suites/generation-quality/cases.jsonl index c619d622..1ffe689c 100644 --- a/evals/suites/generation-quality/cases.jsonl +++ b/evals/suites/generation-quality/cases.jsonl @@ -10,3 +10,6 @@ {"id":"generation-quality.unanswered-spec-questions-ask-user","suite":"generation-quality","executor":"ricky-cli","kind":"regression","input":{"message":"Ricky receives a workflow generation spec with an explicit open question and no `--best-judgement` flag."},"expected":{"ok":false,"contentIncludes":["Generation: failed (status: needs_clarification).","Next: Clarify: Who owns final rollout signoff?"],"forbidPhrases":["Best-judgement clarifications","TypeError","ReferenceError"],"maxToolCalls":1,"must":["Stop before generating a workflow artifact when the spec carries an unanswered question.","Ask the user the unresolved question directly.","Avoid writing an implementation assumption unless the caller explicitly opts into best judgement."],"mustNot":["Generate a workflow by silently guessing the answer.","Hide the clarification question behind a generic failure."],"humanReviewRequired":false},"tags":["generation","clarification","local"],"mock":{"cwd":"temp","specFileContent":"Generate a workflow for package validation.\\nOpen questions:\\n- Who owns final rollout signoff?","argv":"--mode local --spec-file {{specFile}} --no-workforce-persona"}} {"id":"generation-quality.best-judgement-answers-spec-questions","suite":"generation-quality","executor":"ricky-cli","kind":"regression","input":{"message":"Ricky receives the same open-question spec with `--best-judgement`."},"expected":{"ok":true,"contentIncludes":["generated; run when ready","Warning: --best-judgement Who owns final rollout signoff?","Answered by implementing agent impl-primary-codex using --best-judgement","Workflow: workflows/generated/"],"forbidPhrases":["Generation: failed","needs_clarification","TypeError","ReferenceError"],"maxToolCalls":1,"must":["Continue to workflow generation after explicitly answering the unresolved question.","Call out each best-judgement question and answer in user-visible output or generated context.","Identify the implementing agent that made the assumption."],"mustNot":["Pretend the user supplied the answer.","Drop the original question from the assumption record."],"humanReviewRequired":false},"tags":["generation","clarification","local","best-judgement"],"mock":{"cwd":"temp","specFileContent":"Generate a workflow for package validation.\\nOpen questions:\\n- Who owns final rollout signoff?","argv":"local --spec-file {{specFile}} --best-judgement --no-workforce-persona"}} {"id":"generation-quality.mode-local-overrides-runtime-wording","suite":"generation-quality","executor":"ricky-cli","kind":"regression","input":{"message":"Ricky receives a spec that legitimately discusses both local and Cloud execution while the CLI selected local mode."},"expected":{"ok":true,"contentIncludes":["Generation: ok","Run: ricky run workflows/generated/"],"forbidPhrases":["execution-mode-conflict","needs_clarification","Should this workflow run locally/BYOH","TypeError","ReferenceError"],"maxToolCalls":1,"must":["Treat the explicit local CLI mode as the execution preference.","Generate a workflow even when the design spec mentions both local and Cloud runtime support.","Avoid re-asking the local-vs-Cloud clarification after mode has already been chosen."],"mustNot":["Infer `auto` solely from runtime keywords when an explicit CLI mode is present.","Force the user to rewrite a design spec to remove one runtime keyword."],"humanReviewRequired":false},"tags":["generation","clarification","local","issue-76"],"mock":{"cwd":"temp","specFileContent":"Generate a workflow for a primitive whose API supports local BYOH execution and Cloud hosted execution. The generated workflow should implement the primitive docs and validation gates.","argv":"--mode local --spec-file {{specFile}} --no-workforce-persona"}} +{"id":"generation-quality.target-files-from-backticked-prose","suite":"generation-quality","executor":"ricky-cli","kind":"regression","input":{"message":"Ricky receives a markdown spec that names target file paths inside backticks in prose. The parser must recognize them so the workflow targets real source files instead of falling back to the manifest-driven single-artifact path."},"expected":{"ok":true,"contentIncludes":["\"target_files\":","packages/web/app/api/v1/workflows/run/route.ts","packages/core/src/bootstrap/launcher.ts"],"forbidPhrases":["TypeError","ReferenceError"],"maxToolCalls":1,"must":["Extract paths wrapped in markdown backticks into `target_files`.","Surface `target_files` in the generation JSON so callers can verify scope."],"mustNot":["Fall back to the manifest-driven single-artifact path when the spec names concrete files.","Capture prose noise like `base/head` as a target file."],"humanReviewRequired":false},"tags":["generation","target-files","parser","local"],"mock":{"cwd":"temp","specFileContent":"# Spec\\n\\nImplementation plan:\\n\\n- Update `packages/web/app/api/v1/workflows/run/route.ts` to accept the new mode.\\n- Update `packages/core/src/bootstrap/launcher.ts` to provision a sandbox.\\n","argv":"--mode local --spec-file {{specFile}} --no-run --json --no-workforce-persona"}} +{"id":"generation-quality.target-files-from-structured-block","suite":"generation-quality","executor":"ricky-cli","kind":"regression","input":{"message":"A spec with an explicit `## Target Files` block must take precedence over any prose paths so authors can be unambiguous about scope."},"expected":{"ok":true,"contentIncludes":["\"target_files\":","packages/web/app/api/v1/workflows/run/route.ts","packages/core/src/bootstrap/launcher.ts"],"forbidPhrases":["tests/scratch/example.ts","TypeError","ReferenceError"],"maxToolCalls":1,"must":["Honor the structured `## Target Files` block as the source of truth when present.","Strip leading bullets and surrounding backticks from each line in the block."],"mustNot":["Mix prose-extracted candidates into `target_files` when a structured block is declared."],"humanReviewRequired":false},"tags":["generation","target-files","parser","local"],"mock":{"cwd":"temp","specFileContent":"# Spec\\n\\nProse mentions `tests/scratch/example.ts` casually.\\n\\n## Target Files\\n\\n- `packages/web/app/api/v1/workflows/run/route.ts`\\n- packages/core/src/bootstrap/launcher.ts\\n\\n## Acceptance\\n\\nIt works.\\n","argv":"--mode local --spec-file {{specFile}} --no-run --json --no-workforce-persona"}} +{"id":"generation-quality.target-files-suppresses-prose-noise","suite":"generation-quality","executor":"ricky-cli","kind":"regression","input":{"message":"The parser must suppress two-segment prose tokens that have no extension and no recognized leading directory (e.g. `base/head`, `my-org/my-repo`) so they are not captured as target files."},"expected":{"ok":true,"contentIncludes":["\"target_files\":","packages/web/app/api/v1/workflows/run/route.ts"],"forbidPhrases":["\"\\\"base/head\\\"\"","\"\\\"user/account\\\"\"","TypeError","ReferenceError"],"maxToolCalls":1,"must":["Keep real backticked paths in `target_files`.","Drop two-segment prose tokens that look like noise."],"mustNot":["Capture human-readable phrases as file paths."],"humanReviewRequired":false},"tags":["generation","target-files","parser","local"],"mock":{"cwd":"temp","specFileContent":"# Spec\\n\\nSend the PR number, base/head SHA, and the user/account pair to MSD. Then update `packages/web/app/api/v1/workflows/run/route.ts`.\\n","argv":"--mode local --spec-file {{specFile}} --no-run --json --no-workforce-persona"}} diff --git a/evals/suites/generation-quality/cases.md b/evals/suites/generation-quality/cases.md index c4688101..fd0d66f0 100644 --- a/evals/suites/generation-quality/cases.md +++ b/evals/suites/generation-quality/cases.md @@ -263,3 +263,102 @@ maxToolCalls: 1 ### Must Not - Infer `auto` solely from runtime keywords when an explicit CLI mode is present. - Force the user to rewrite a design spec to remove one runtime keyword. +## generation-quality.target-files-from-backticked-prose +Executor: ricky-cli +Kind: regression +Tags: generation, target-files, parser, local +Human Review: false + +### Message +Ricky receives a markdown spec that names target file paths inside backticks in prose. The parser must recognize them so the workflow targets real source files instead of falling back to the manifest-driven single-artifact path. + +### Mock +cwd: temp +specFileContent: # Spec\n\nImplementation plan:\n\n- Update `packages/web/app/api/v1/workflows/run/route.ts` to accept the new mode.\n- Update `packages/core/src/bootstrap/launcher.ts` to provision a sandbox.\n +argv: --mode local --spec-file {{specFile}} --no-run --json --no-workforce-persona + +### Deterministic Checks +ok: true +contentIncludes: +- "target_files": +- packages/web/app/api/v1/workflows/run/route.ts +- packages/core/src/bootstrap/launcher.ts +forbidPhrases: +- TypeError +- ReferenceError +maxToolCalls: 1 + +### Must +- Extract paths wrapped in markdown backticks into `target_files`. +- Surface `target_files` in the generation JSON so callers can verify scope. + +### Must Not +- Fall back to the manifest-driven single-artifact path when the spec names concrete files. +- Capture prose noise like `base/head` as a target file. + +## generation-quality.target-files-from-structured-block +Executor: ricky-cli +Kind: regression +Tags: generation, target-files, parser, local +Human Review: false + +### Message +A spec with an explicit `## Target Files` block must take precedence over any prose paths so authors can be unambiguous about scope. + +### Mock +cwd: temp +specFileContent: # Spec\n\nProse mentions `tests/scratch/example.ts` casually.\n\n## Target Files\n\n- `packages/web/app/api/v1/workflows/run/route.ts`\n- packages/core/src/bootstrap/launcher.ts\n\n## Acceptance\n\nIt works.\n +argv: --mode local --spec-file {{specFile}} --no-run --json --no-workforce-persona + +### Deterministic Checks +ok: true +contentIncludes: +- "target_files": +- packages/web/app/api/v1/workflows/run/route.ts +- packages/core/src/bootstrap/launcher.ts +forbidPhrases: +- tests/scratch/example.ts +- TypeError +- ReferenceError +maxToolCalls: 1 + +### Must +- Honor the structured `## Target Files` block as the source of truth when present. +- Strip leading bullets and surrounding backticks from each line in the block. + +### Must Not +- Mix prose-extracted candidates into `target_files` when a structured block is declared. + +## generation-quality.target-files-suppresses-prose-noise +Executor: ricky-cli +Kind: regression +Tags: generation, target-files, parser, local +Human Review: false + +### Message +The parser must suppress two-segment prose tokens that have no extension and no recognized leading directory (e.g. `base/head`, `my-org/my-repo`) so they are not captured as target files. + +### Mock +cwd: temp +specFileContent: # Spec\n\nSend the PR number, base/head SHA, and the user/account pair to MSD. Then update `packages/web/app/api/v1/workflows/run/route.ts`.\n +argv: --mode local --spec-file {{specFile}} --no-run --json --no-workforce-persona + +### Deterministic Checks +ok: true +contentIncludes: +- "target_files": +- packages/web/app/api/v1/workflows/run/route.ts +forbidPhrases: +- "\"base/head\"" +- "\"user/account\"" +- TypeError +- ReferenceError +maxToolCalls: 1 + +### Must +- Keep real backticked paths in `target_files`. +- Drop two-segment prose tokens that look like noise. + +### Must Not +- Capture human-readable phrases as file paths. + diff --git a/package-lock.json b/package-lock.json index 4764615e..6f3725f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@agentworkforce/harness-kit": "^0.19.0", "@agentworkforce/workload-router": "^0.19.0", "@inquirer/prompts": "^8.4.2", + "mdast-util-from-markdown": "^2.0.3", "ora": "^8.2.0", "ssh2": "^1.17.0" }, @@ -23,6 +24,7 @@ }, "devDependencies": { "@agent-assistant/telemetry": "^0.4.31", + "@types/mdast": "^4.0.4", "@types/node": "^24.5.2", "esbuild": "^0.28.0", "tsx": "^4.21.0", @@ -3931,6 +3933,15 @@ "assertion-error": "^2.0.1" } }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -3945,6 +3956,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.12.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", @@ -3960,6 +3986,12 @@ "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", "license": "MIT" }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, "node_modules/@vitest/expect": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", @@ -4250,6 +4282,16 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chardet": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", @@ -4384,7 +4426,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -4398,6 +4439,19 @@ } } }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/deep-eql": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", @@ -4417,6 +4471,28 @@ "node": ">=0.4.0" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -5081,6 +5157,485 @@ "node": ">= 0.4" } }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -5139,7 +5694,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/mute-stream": { @@ -6278,6 +6832,19 @@ "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/vite": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", diff --git a/package.json b/package.json index 37f53923..d84de385 100644 --- a/package.json +++ b/package.json @@ -62,11 +62,13 @@ "@agentworkforce/harness-kit": "^0.19.0", "@agentworkforce/workload-router": "^0.19.0", "@inquirer/prompts": "^8.4.2", + "mdast-util-from-markdown": "^2.0.3", "ora": "^8.2.0", "ssh2": "^1.17.0" }, "devDependencies": { "@agent-assistant/telemetry": "^0.4.31", + "@types/mdast": "^4.0.4", "@types/node": "^24.5.2", "esbuild": "^0.28.0", "tsx": "^4.21.0", diff --git a/src/local/entrypoint.ts b/src/local/entrypoint.ts index 97b5dc05..1e154b80 100644 --- a/src/local/entrypoint.ts +++ b/src/local/entrypoint.ts @@ -86,6 +86,7 @@ export interface LocalGenerationStageResult { path: string; workflow_id: string; spec_digest: string; + target_files?: string[]; }; next?: { run_command: string; @@ -1092,6 +1093,7 @@ export function createLocalExecutor(options: LocalExecutorOptions = {}): LocalEx generationResult, assistantTurnContext, bestJudgementClarifications, + normalizedSpec.targetFiles, ); return { ok: false, @@ -1112,7 +1114,7 @@ export function createLocalExecutor(options: LocalExecutorOptions = {}): LocalEx await writeGenerationMetadataArtifacts(generationResult, artifactWriter, cwd); } logs.push(`[local] wrote workflow artifact: ${artifact.artifactPath}`); - generationStage = createGenerationStage('ok', artifact, specDigest, undefined, generationResult, assistantTurnContext, bestJudgementClarifications); + generationStage = createGenerationStage('ok', artifact, specDigest, undefined, generationResult, assistantTurnContext, bestJudgementClarifications, normalizedSpec.targetFiles); } const runTarget = artifact?.artifactPath ?? workflowFile; @@ -1653,6 +1655,7 @@ function createGenerationStage( generationResult?: GenerationResult, assistantTurnContext?: LocalAssistantTurnContextDecision, bestJudgementClarifications?: BestJudgementClarificationDecision[], + targetFiles?: string[], ): LocalGenerationStageResult { return { stage: 'generate', @@ -1663,6 +1666,7 @@ function createGenerationStage( path: artifact.artifactPath, workflow_id: artifact.workflowId, spec_digest: specDigest, + ...(targetFiles && targetFiles.length > 0 ? { target_files: targetFiles } : {}), }, } : {}), diff --git a/src/product/spec-intake/markdown-target-files.test.ts b/src/product/spec-intake/markdown-target-files.test.ts new file mode 100644 index 00000000..65e0c32e --- /dev/null +++ b/src/product/spec-intake/markdown-target-files.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; + +import { extractTargetFilesFromMarkdown, looksLikeRealPath } from './markdown-target-files.js'; + +describe('extractTargetFilesFromMarkdown', () => { + it('returns inline-code paths from prose', () => { + expect( + extractTargetFilesFromMarkdown( + 'Edit `packages/web/foo.ts` and `src/lib/bar.ts` to fix the bug.', + ), + ).toEqual(['packages/web/foo.ts', 'src/lib/bar.ts']); + }); + + it('skips inline-code tokens that fail the path filter', () => { + expect( + extractTargetFilesFromMarkdown( + 'Run `npm install` then rebase onto `git/main`. Edit `packages/web/foo.ts`.', + ), + ).toEqual(['packages/web/foo.ts']); + }); + + it('does not capture paths inside fenced code blocks', () => { + const text = [ + 'Edit `packages/web/foo.ts`.', + '', + '```ts', + 'const x = "packages/legacy/old.ts";', + '```', + ].join('\n'); + expect(extractTargetFilesFromMarkdown(text)).toEqual(['packages/web/foo.ts']); + }); + + it('honors a `## Target Files` block over inline code', () => { + const text = [ + 'Some prose with `tests/scratch.ts`.', + '', + '## Target Files', + '', + '- `packages/web/route.ts`', + '- packages/core/launcher.ts', + '', + '## Acceptance', + ].join('\n'); + expect(extractTargetFilesFromMarkdown(text)).toEqual([ + 'packages/web/route.ts', + 'packages/core/launcher.ts', + ]); + }); + + it('matches the Target Files heading at any depth', () => { + const text = [ + '## Plan', + '', + '### Target Files', + '', + '- `packages/web/route.ts`', + ].join('\n'); + expect(extractTargetFilesFromMarkdown(text)).toEqual(['packages/web/route.ts']); + }); + + it('returns an empty array when given non-markdown plain text with no inline code', () => { + expect(extractTargetFilesFromMarkdown('Plain text. No paths anywhere.')).toEqual([]); + }); + + it('deduplicates repeated mentions of the same path', () => { + expect( + extractTargetFilesFromMarkdown( + 'Edit `packages/web/foo.ts` and again `packages/web/foo.ts` for emphasis.', + ), + ).toEqual(['packages/web/foo.ts']); + }); + + it('rejects http(s) URLs that happen to live in inline code', () => { + expect( + extractTargetFilesFromMarkdown('See `https://example.com/api/v1/foo` for context.'), + ).toEqual([]); + }); + + it('rejects URLs and prose noise inside a `## Target Files` block', () => { + const text = [ + '## Target Files', + '', + '- `packages/web/route.ts`', + '- https://example.com/api/v1/foo', + '- base/head', + '- `packages/core/launcher.ts`,', + '', + '## Acceptance', + ].join('\n'); + expect(extractTargetFilesFromMarkdown(text)).toEqual([ + 'packages/web/route.ts', + 'packages/core/launcher.ts', + ]); + }); +}); + +describe('looksLikeRealPath', () => { + it('accepts paths with extensions', () => { + expect(looksLikeRealPath('packages/web/foo.ts')).toBe(true); + expect(looksLikeRealPath('src/index.tsx')).toBe(true); + }); + + it('accepts deeply-nested paths without extensions', () => { + expect(looksLikeRealPath('packages/web/lib/nango-bridge')).toBe(true); + }); + + it('accepts two-segment paths only when they have a recognized prefix', () => { + expect(looksLikeRealPath('workflows/wave2-product')).toBe(true); + expect(looksLikeRealPath('packages/web')).toBe(true); + expect(looksLikeRealPath('base/head')).toBe(false); + expect(looksLikeRealPath('git/main')).toBe(false); + expect(looksLikeRealPath('my-org/my-repo')).toBe(false); + }); + + it('rejects empty, whitespace-bearing, or http(s) tokens', () => { + expect(looksLikeRealPath('')).toBe(false); + expect(looksLikeRealPath('foo bar/baz.ts')).toBe(false); + expect(looksLikeRealPath('http://example.com/foo.ts')).toBe(false); + expect(looksLikeRealPath('https://example.com/foo.ts')).toBe(false); + }); + + it('rejects single-segment tokens (no slash)', () => { + expect(looksLikeRealPath('foo.ts')).toBe(false); + expect(looksLikeRealPath('parseSpec')).toBe(false); + }); +}); diff --git a/src/product/spec-intake/markdown-target-files.ts b/src/product/spec-intake/markdown-target-files.ts new file mode 100644 index 00000000..579b7373 --- /dev/null +++ b/src/product/spec-intake/markdown-target-files.ts @@ -0,0 +1,154 @@ +// Extract target file paths from a markdown spec via mdast. +// +// Why an AST instead of regex: backticks are markdown's native syntax for +// "this token is code/path-like". Walking the parsed tree and collecting +// `inlineCode` nodes is exact — no boundary char-class to keep growing, no +// false matches from prose fragments like `base/head`. Fenced code blocks +// (```...```) are excluded by construction since they're `code` nodes, not +// `inlineCode`, so example paths in code blocks aren't mistaken for edits. + +import { fromMarkdown } from 'mdast-util-from-markdown'; +import type { + Heading, + InlineCode, + List, + ListItem, + Node, + Paragraph, + Parent, + Root, +} from 'mdast'; + +const RECOGNIZED_PATH_PREFIXES = [ + 'packages/', + 'src/', + 'tests/', + 'test/', + 'lib/', + 'app/', + 'apps/', + 'bin/', + 'scripts/', + 'infra/', + 'docs/', + 'workflows/', + 'migrations/', + 'examples/', + 'e2e/', + 'public/', + 'static/', + 'assets/', + 'config/', + 'cmd/', + 'internal/', + 'pkg/', + '.github/', + '.claude/', + '.agents/', +]; + +export function extractTargetFilesFromMarkdown(text: string): string[] { + let tree: Root; + try { + tree = fromMarkdown(text); + } catch { + return []; + } + const fromBlock = extractFromTargetFilesSection(tree); + if (fromBlock.length > 0) return fromBlock; + return extractFromInlineCode(tree); +} + +export function looksLikeRealPath(candidate: string): boolean { + if (!candidate || candidate.includes(' ')) return false; + if (candidate.startsWith('http')) return false; + if (!candidate.includes('/')) return false; + if (/\.[A-Za-z0-9]+$/.test(candidate)) return true; + const slashes = (candidate.match(/\//g) ?? []).length; + if (slashes >= 2) return true; + return RECOGNIZED_PATH_PREFIXES.some((prefix) => candidate.startsWith(prefix)); +} + +function extractFromTargetFilesSection(tree: Root): string[] { + const out: string[] = []; + let inSection = false; + let sectionDepth = 0; + for (const child of tree.children) { + if (child.type === 'heading') { + const heading = child as Heading; + const headingText = collectText(heading).trim(); + if (inSection && heading.depth <= sectionDepth) { + break; + } + if (/^target\s+files\s*$/i.test(headingText)) { + inSection = true; + sectionDepth = heading.depth; + } + continue; + } + if (!inSection) continue; + if (child.type === 'list') { + for (const item of (child as List).children) { + const path = pathFromListItem(item); + if (!path) continue; + const cleaned = path.replace(/[`'")\],.;:]+$/, '').trim(); + if (looksLikeRealPath(cleaned)) out.push(cleaned); + } + } + } + return dedupe(out); +} + +function pathFromListItem(item: ListItem): string | null { + for (const child of item.children) { + if (child.type === 'paragraph') { + const para = child as Paragraph; + const inline = para.children.find( + (c): c is InlineCode => c.type === 'inlineCode', + ); + if (inline) return inline.value.trim(); + const txt = collectText(para).trim(); + if (txt) return stripWrappers(txt); + } + } + return null; +} + +function extractFromInlineCode(tree: Root): string[] { + const out: string[] = []; + walk(tree, (node) => { + if (node.type !== 'inlineCode') return; + const value = (node as InlineCode).value.trim(); + if (looksLikeRealPath(value)) out.push(value); + }); + return dedupe(out); +} + +function walk(node: Node, visit: (node: Node) => void): void { + visit(node); + if (isParent(node)) { + for (const child of node.children) walk(child, visit); + } +} + +function isParent(node: Node): node is Parent { + return Array.isArray((node as Parent).children); +} + +function collectText(node: Node): string { + if (node.type === 'code') return ''; + const candidate = (node as unknown as { value?: unknown }).value; + if (typeof candidate === 'string') return candidate; + if (isParent(node)) { + return node.children.map(collectText).join(''); + } + return ''; +} + +function stripWrappers(value: string): string { + return value.replace(/^[`'"]|[`'"]$/g, '').trim(); +} + +function dedupe(values: string[]): string[] { + return Array.from(new Set(values)); +} diff --git a/src/product/spec-intake/parser.test.ts b/src/product/spec-intake/parser.test.ts index 3b10d3dc..adb79bab 100644 --- a/src/product/spec-intake/parser.test.ts +++ b/src/product/spec-intake/parser.test.ts @@ -669,4 +669,142 @@ describe('spec intake parser, normalizer, and router', () => { }), ]); }); + + describe('targetFiles extraction (AST-driven)', () => { + it('captures backticked paths from markdown prose via inlineCode nodes', () => { + const result = intake( + natural( + [ + 'Implementation plan:', + '- Update `packages/web/app/api/v1/workflows/run/route.ts` to accept the new mode.', + '- Update `packages/core/src/bootstrap/launcher.ts` to provision a sandbox.', + ].join('\n'), + ), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toEqual( + expect.arrayContaining([ + 'packages/web/app/api/v1/workflows/run/route.ts', + 'packages/core/src/bootstrap/launcher.ts', + ]), + ); + }); + + it('does not capture paths inside fenced code blocks', () => { + const result = intake( + natural( + [ + 'Edit `packages/web/route.ts` to fix the bug.', + '', + 'The current code looks like:', + '', + '```ts', + 'import { foo } from "packages/legacy/old-helper.ts";', + 'export const bar = "examples/sample/path.json";', + '```', + '', + 'Replace it.', + ].join('\n'), + ), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toContain('packages/web/route.ts'); + expect(targets).not.toContain('packages/legacy/old-helper.ts'); + expect(targets).not.toContain('examples/sample/path.json'); + }); + + it('suppresses backticked tokens that are not paths (git refs, type names)', () => { + const result = intake( + natural( + [ + 'Touch `packages/web/route.ts` and rebase onto `git/main`.', + 'The new type `MsdRelayReviewArtifact` lives in `packages/core/types.ts`.', + ].join('\n'), + ), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toContain('packages/web/route.ts'); + expect(targets).toContain('packages/core/types.ts'); + // `git/main` has 2 segments, no extension, no recognized prefix → filtered. + expect(targets).not.toContain('git/main'); + // `MsdRelayReviewArtifact` has no slash → never considered. + expect(targets).not.toContain('MsdRelayReviewArtifact'); + }); + + it('extracts paths from a structured `## Target Files` block in priority over inline code', () => { + const result = intake( + natural( + [ + '# Spec', + 'Some prose mentioning `tests/scratch/example.ts` casually.', + '', + '## Target Files', + '', + '- `packages/web/app/api/v1/workflows/run/route.ts`', + '- packages/core/src/bootstrap/launcher.ts', + '', + '## Acceptance', + 'It works.', + ].join('\n'), + ), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toEqual([ + 'packages/web/app/api/v1/workflows/run/route.ts', + 'packages/core/src/bootstrap/launcher.ts', + ]); + expect(targets).not.toContain('tests/scratch/example.ts'); + }); + + it('honors a `### Target Files` block at any heading depth', () => { + const result = intake( + natural( + [ + '## Implementation', + '', + '### Target Files', + '', + '- `packages/web/route.ts`', + '', + '### Tests', + '', + 'Run vitest.', + ].join('\n'), + ), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toEqual(['packages/web/route.ts']); + }); + + it('falls back to inline code extraction when there is no Target Files block', () => { + const result = intake( + natural('Edit `packages/core/src/bootstrap/launcher.ts` to provision a sandbox.'), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toContain('packages/core/src/bootstrap/launcher.ts'); + }); + + it('suppresses prose noise in non-markdown inputs via the regex fallback', () => { + // No backticks anywhere — the AST extractor returns []. The plain-text + // fallback then applies looksLikeRealPath to drop noise. + const result = intake( + natural( + 'Send the PR number, base/head SHA, and the user/account pair to MSD. ' + + 'Update packages/web/route.ts.', + ), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toContain('packages/web/route.ts'); + expect(targets).not.toContain('base/head'); + expect(targets).not.toContain('user/account'); + }); + + it('keeps deeply-nested paths without an extension (3+ segments)', () => { + const result = intake( + natural('Touch `packages/web/lib/nango-bridge` for the new adapter.'), + ); + const targets = result.routing?.normalizedSpec.targetFiles ?? []; + expect(targets).toContain('packages/web/lib/nango-bridge'); + }); + }); }); diff --git a/src/product/spec-intake/parser.ts b/src/product/spec-intake/parser.ts index 87146216..9b7e9f12 100644 --- a/src/product/spec-intake/parser.ts +++ b/src/product/spec-intake/parser.ts @@ -11,6 +11,7 @@ import type { } from './types.js'; import type { Confidence } from '../../runtime/failure/types.js'; +import { extractTargetFilesFromMarkdown, looksLikeRealPath } from './markdown-target-files.js'; interface ExtractedFields { description: string; @@ -114,7 +115,10 @@ const MCP_TOOL_INTENTS: Record = { 'ricky.workflow.execute': 'execute', }; -const PATH_PATTERN = /(?:^|\s)([./~]?[\w@.-]+(?:\/[\w@.-]+)+(?:\.[A-Za-z0-9]+)?)/g; +// Plain-text path scan, used as a defensive fallback for non-markdown-shaped +// inputs (Slack messages, raw JSON description blobs). For markdown specs the +// AST-based extractor in `./markdown-target-files.ts` is the source of truth. +const PATH_PATTERN = /(?:^|[\s`'"(\[<])([./~]?[\w@.-]+(?:\/[\w@.-]+)+(?:\.[A-Za-z0-9]+)?)/g; const DESCRIPTION_KEYS = [ 'description', 'prompt', @@ -849,12 +853,20 @@ function extractTargetContext(text: string): string | undefined { } function extractTargetFiles(text: string): string[] { + const fromAst = extractTargetFilesFromMarkdown(text); + if (fromAst.length > 0) return fromAst; + // Fallback for non-markdown inputs (Slack one-liners, MCP description blobs). + // The AST extractor returns [] when the input has no inline-code spans and no + // Target Files section, so we fall through to a plain-text scan that still + // applies `looksLikeRealPath` to suppress prose noise like `base/head`. const paths: string[] = []; for (const match of text.matchAll(PATH_PATTERN)) { const candidate = match[1]; - if (candidate && !candidate.startsWith('http')) { - paths.push(candidate.replace(/[),.;:]$/, '')); - } + if (!candidate) continue; + const cleaned = candidate.replace(/[`'")\],.;:]+$/, ''); + if (!cleaned || cleaned.startsWith('http')) continue; + if (!looksLikeRealPath(cleaned)) continue; + paths.push(cleaned); } return dedupe(paths); }