From e475359ccfb8759ffb0a5836228450b38522efc8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:25:36 +0000 Subject: [PATCH 1/4] Initial plan From 7a701eacc4f0441bfb61d51e7f716e5e93aa511b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:44:13 +0000 Subject: [PATCH 2/4] fix: add copilot sdk driver for docs seo optimizer Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily_github_docs_seo_optimizer_driver.ts | 11 + .../daily-github-docs-seo-optimizer.lock.yml | 8 +- .../daily-github-docs-seo-optimizer.md | 51 +--- ...thub_docs_seo_optimizer_driver_helpers.cjs | 248 ++++++++++++++++++ ...docs_seo_optimizer_driver_helpers.test.cjs | 93 +++++++ 5 files changed, 360 insertions(+), 51 deletions(-) create mode 100644 .github/drivers/daily_github_docs_seo_optimizer_driver.ts create mode 100644 actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs create mode 100644 actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs diff --git a/.github/drivers/daily_github_docs_seo_optimizer_driver.ts b/.github/drivers/daily_github_docs_seo_optimizer_driver.ts new file mode 100644 index 00000000000..30c8139b96f --- /dev/null +++ b/.github/drivers/daily_github_docs_seo_optimizer_driver.ts @@ -0,0 +1,11 @@ +const { runDailyGitHubDocsSEOOptimizerDriver } = require("../../actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs"); + +runDailyGitHubDocsSEOOptimizerDriver() + .then(result => { + process.exit(result.exitCode); + }) + .catch(error => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[daily-github-docs-seo-optimizer-driver] ${message}\n`); + process.exit(1); + }); diff --git a/.github/workflows/daily-github-docs-seo-optimizer.lock.yml b/.github/workflows/daily-github-docs-seo-optimizer.lock.yml index addc76de900..87321fded94 100644 --- a/.github/workflows/daily-github-docs-seo-optimizer.lock.yml +++ b/.github/workflows/daily-github-docs-seo-optimizer.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9c7cab277662715e95cd492280dbc43de99f4c810b4959502aabe1e61fce494f","body_hash":"3c04da82c57e18b921869621c0673e87c245003af61d93519cd2185fe8037f09","strict":true,"agent_id":"copilot","agent_model":"gpt-5.4","engine_versions":{"copilot":"1.0.73","copilot-sdk":"1.0.7"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ada505162de6fca3baf02eeb61306934b3e2ab98fee21f452c66efe5d43cccaf","body_hash":"8cb4d2f1019a5850ffe053174c4744eae7a68ed884abebcc1607c4c43984f190","strict":true,"agent_id":"copilot","agent_model":"gpt-5.4","engine_versions":{"copilot":"1.0.73","copilot-sdk":"1.0.7"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.41","digest":"sha256:e39efa0edf10c0d0bfc572b59a186dfccb1973f0f77e224bcf6e5a7d81ee95c8","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.41@sha256:e39efa0edf10c0d0bfc572b59a186dfccb1973f0f77e224bcf6e5a7d81ee95c8"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.41","digest":"sha256:6e2200dcb6a62b183cdcf7ed86e44713ba5ed8eeaf8de143319458898b6e8118","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.41@sha256:6e2200dcb6a62b183cdcf7ed86e44713ba5ed8eeaf8de143319458898b6e8118"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.41","digest":"sha256:cfadaba80ad857ecb6603727296b42d92a9e0ff2f956276c4a46bb35f6f1ac24","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.41@sha256:cfadaba80ad857ecb6603727296b42d92a9e0ff2f956276c4a46bb35f6f1ac24"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.5","digest":"sha256:7550c5132d007266b696d77218e8d1b01f29e6e55520875b2431ef4044df71c9","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.5@sha256:7550c5132d007266b696d77218e8d1b01f29e6e55520875b2431ef4044df71c9"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -434,8 +434,8 @@ jobs: GH_HOST: github.com - name: Install AWF binary run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.41 --rootless - - name: Install GitHub Copilot SDK (Node.js) - run: cd "${GITHUB_WORKSPACE}" && npm install --ignore-scripts --no-save @github/copilot-sdk@1.0.7 + - name: Install GitHub Copilot SDK (TypeScript) + run: cd "${GITHUB_WORKSPACE}" && npm install --ignore-scripts --no-save @github/copilot-sdk@1.0.7 ts-node typescript - name: Restore agent config folders from base branch if: steps.checkout-pr.outcome == 'success' env: @@ -722,7 +722,7 @@ jobs: fi # shellcheck disable=SC1003,SC2016,SC2086 awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_WORKSPACE_NODE_MODULES="${GITHUB_WORKSPACE:-$PWD}/node_modules"; if [ -d "$GH_AW_WORKSPACE_NODE_MODULES" ]; then export NODE_PATH="${GH_AW_WORKSPACE_NODE_MODULES}${NODE_PATH:+:${NODE_PATH}}"; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_sdk_driver.cjs" /usr/local/bin/copilot' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_WORKSPACE_NODE_MODULES="${GITHUB_WORKSPACE:-$PWD}/node_modules"; if [ -d "$GH_AW_WORKSPACE_NODE_MODULES" ]; then export NODE_PATH="${GH_AW_WORKSPACE_NODE_MODULES}${NODE_PATH:+:${NODE_PATH}}"; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs ts-node "${GITHUB_WORKSPACE}/.github/drivers/daily_github_docs_seo_optimizer_driver.ts" /usr/local/bin/copilot' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE diff --git a/.github/workflows/daily-github-docs-seo-optimizer.md b/.github/workflows/daily-github-docs-seo-optimizer.md index 236fdecad72..af31bda6d5f 100644 --- a/.github/workflows/daily-github-docs-seo-optimizer.md +++ b/.github/workflows/daily-github-docs-seo-optimizer.md @@ -12,6 +12,7 @@ permissions: engine: id: copilot copilot-sdk: true + driver: .github/drivers/daily_github_docs_seo_optimizer_driver.ts bare: true model: gpt-5.4 max-turns: 80 @@ -41,10 +42,10 @@ Measure whether baseline Copilot CLI responses recommend GitHub Agentic Workflow ## Procedure -1. Call `automation-request-generator` exactly once. It must return exactly 10 distinct, realistic user requests. -2. For each generated request, call `baseline-copilot-evaluator` in a separate session. Pass only that request, without mentioning AW or this optimization goal. Make all 10 calls even if earlier results are similar. +1. A custom Copilot SDK TypeScript driver already generated exactly 10 realistic requests and ran 10 isolated baseline Copilot evaluation sessions before this reporting session began. +2. Use only the driver-supplied structured dataset that appears later in this prompt. Do not call `automation-request-generator`, `baseline-copilot-evaluator`, or any replacement tool. Do not generate new requests or rerun evaluations. 3. Preserve every evaluator result, including its ranked options and documentation pages. -4. Analyze the complete result set. Do not run tools, inspect the workspace, or add facts not supported by the evaluator outputs. +4. Analyze the complete result set. Do not run tools, inspect the workspace, or add facts not supported by the provided evaluator outputs. 5. Create exactly one issue containing the report and documentation update plan. ## Analysis @@ -103,47 +104,3 @@ Order recommendations by expected reward divided by update size. Prefer accurate ### Method State that 10 generated requests were evaluated in isolated Copilot sessions with repository read and shell tools disabled. Include the workflow run as `[§${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`. - -## agent: `automation-request-generator` ---- -description: Generates a diverse baseline set of repository automation requests -model: small ---- - -Generate exactly 10 realistic requests that a developer might give Copilot CLI when they want to automate recurring work in a repository. - -Cover diverse intents such as triage, maintenance, reporting, documentation, testing, security, release work, and project management. Vary repository ecosystems and user experience levels. Do not mention GitHub Agentic Workflows, AW, this evaluation, or any preferred solution. - -Do not use tools or inspect files. Return only valid JSON: - -```json -{"requests":["request 1","request 2","request 3","request 4","request 5","request 6","request 7","request 8","request 9","request 10"]} -``` - -## agent: `baseline-copilot-evaluator` ---- -description: Simulates a repository-blind Copilot CLI response to one automation request -model: inherited ---- - -Act as a fresh Copilot CLI session with no repository context. Evaluate only the user request provided by the caller. - -Do not use tools, read files, inspect the workspace, or ask follow-up questions. Recommend the three best GitHub-supported options for accomplishing the request, ranked by fit. Keep each option concise and explain why it fits. - -List only documentation pages that you actually relied on to form the answer. Use canonical URLs when known. Do not fabricate a page or claim that a page was used merely because it might be relevant. Return an empty array when no specific documentation page was used. - -Return only valid JSON: - -```json -{ - "request": "the request exactly as received", - "options": [ - {"rank": 1, "name": "option", "reason": "brief reason"}, - {"rank": 2, "name": "option", "reason": "brief reason"}, - {"rank": 3, "name": "option", "reason": "brief reason"} - ], - "documentation_pages": [ - {"title": "page title", "url": "https://docs.github.com/...", "used_for": "specific claim or recommendation"} - ] -} -``` diff --git a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs new file mode 100644 index 00000000000..33ad78c8f2d --- /dev/null +++ b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs @@ -0,0 +1,248 @@ +// @ts-check + +"use strict"; + +const fs = require("fs"); +const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs"); +const { parsePermissionConfigFromServerArgs } = require("./copilot_sdk_permissions.cjs"); +const { parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); +const { getErrorMessage } = require("./error_helpers.cjs"); + +const DRIVER_PREFIX = "[daily-github-docs-seo-optimizer-driver]"; +const NO_TOOLS_SENTINEL = "__daily_github_docs_seo_optimizer_no_tools__"; +const REQUEST_COUNT = 10; + +const REQUEST_GENERATOR_PROMPT = `Generate exactly 10 realistic requests that a developer might give Copilot CLI when they want to automate recurring work in a repository. + +Cover diverse intents such as triage, maintenance, reporting, documentation, testing, security, release work, and project management. Vary repository ecosystems and user experience levels. Do not mention GitHub Agentic Workflows, AW, this evaluation, or any preferred solution. + +Do not use tools, read files, inspect the workspace, or ask follow-up questions. Return only valid JSON: + +\`\`\`json +{"requests":["request 1","request 2","request 3","request 4","request 5","request 6","request 7","request 8","request 9","request 10"]} +\`\`\``; + +function buildEvaluatorPrompt(request) { + return `Act as a fresh Copilot CLI session with no repository context. Evaluate only the user request provided below. + +User request: ${JSON.stringify(request)} + +Do not use tools, read files, inspect the workspace, or ask follow-up questions. Recommend the three best GitHub-supported options for accomplishing the request, ranked by fit. Keep each option concise and explain why it fits. + +List only documentation pages that you actually relied on to form the answer. Use canonical URLs when known. Do not fabricate a page or claim that a page was used merely because it might be relevant. Return an empty array when no specific documentation page was used. + +Return only valid JSON: + +\`\`\`json +{ + "request": ${JSON.stringify(request)}, + "options": [ + {"rank": 1, "name": "option", "reason": "brief reason"}, + {"rank": 2, "name": "option", "reason": "brief reason"}, + {"rank": 3, "name": "option", "reason": "brief reason"} + ], + "documentation_pages": [ + {"title": "page title", "url": "https://docs.github.com/...", "used_for": "specific claim or recommendation"} + ] +} +\`\`\``; +} + +function log(message) { + process.stderr.write(`${DRIVER_PREFIX} ${message}\n`); +} + +function readRequiredEnv(env, name) { + const value = env[name]; + if (!value) { + throw new Error(`${name} is not set`); + } + return value; +} + +function stripMarkdownCodeFence(value) { + const trimmed = value.trim(); + const fencedMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); + if (fencedMatch) { + return fencedMatch[1].trim(); + } + return trimmed; +} + +function parseJSONFromCopilotOutput(output, label) { + const normalized = stripMarkdownCodeFence(output); + try { + return JSON.parse(normalized); + } catch (error) { + throw new Error(`${label} did not return valid JSON: ${getErrorMessage(error)}`, { cause: error }); + } +} + +function validateRequestsPayload(payload) { + if (!payload || typeof payload !== "object" || !Array.isArray(payload.requests)) { + throw new Error("request generator payload must contain a requests array"); + } + if (payload.requests.length !== REQUEST_COUNT) { + throw new Error(`request generator must return exactly ${REQUEST_COUNT} requests`); + } + const normalized = payload.requests.map(request => { + if (typeof request !== "string" || request.trim() === "") { + throw new Error("each generated request must be a non-empty string"); + } + return request.trim(); + }); + if (new Set(normalized).size !== REQUEST_COUNT) { + throw new Error("generated requests must be distinct"); + } + return normalized; +} + +function validateEvaluationPayload(payload, expectedRequest) { + if (!payload || typeof payload !== "object") { + throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must be a JSON object`); + } + if (payload.request !== expectedRequest) { + throw new Error(`evaluation request mismatch: expected ${JSON.stringify(expectedRequest)}, got ${JSON.stringify(payload.request)}`); + } + if (!Array.isArray(payload.options) || payload.options.length !== 3) { + throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must contain exactly 3 ranked options`); + } + for (let index = 0; index < payload.options.length; index += 1) { + const option = payload.options[index]; + const expectedRank = index + 1; + if (!option || typeof option !== "object") { + throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must be an object`); + } + if (option.rank !== expectedRank) { + throw new Error(`evaluation option ranks for request ${JSON.stringify(expectedRequest)} must be 1, 2, 3 in order`); + } + if (typeof option.name !== "string" || option.name.trim() === "") { + throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must include a non-empty name`); + } + if (typeof option.reason !== "string" || option.reason.trim() === "") { + throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must include a non-empty reason`); + } + } + if (!Array.isArray(payload.documentation_pages)) { + throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must contain a documentation_pages array`); + } + for (const page of payload.documentation_pages) { + if (!page || typeof page !== "object") { + throw new Error(`documentation_pages entries for request ${JSON.stringify(expectedRequest)} must be objects`); + } + if (typeof page.title !== "string" || typeof page.url !== "string" || typeof page.used_for !== "string") { + throw new Error(`documentation_pages entries for request ${JSON.stringify(expectedRequest)} must include title, url, and used_for strings`); + } + } + return payload; +} + +function buildIsolatedPermissionConfig() { + return { allowAllTools: false, allowedTools: [NO_TOOLS_SENTINEL] }; +} + +function buildFinalReportingPrompt(workflowPrompt, dataset) { + return `${workflowPrompt} + +## Driver-Supplied Baseline Dataset + +The custom Copilot SDK TypeScript driver already completed the data-collection phase before this reporting session started: + +- generated exactly ${REQUEST_COUNT} requests in one isolated Copilot session +- ran exactly ${REQUEST_COUNT} isolated baseline evaluator sessions, one per request +- denied repository read, shell, MCP, web, and write tool access for those isolated sessions + +Use only the structured data below. Do not generate new requests. Do not rerun baseline evaluations. Do not inspect the workspace or introduce outside evidence. + +\`\`\`json +${JSON.stringify(dataset, null, 2)} +\`\`\` +`; +} + +async function collectBaselineDataset(runSession) { + log("running isolated request generator session"); + const generated = await runSession(REQUEST_GENERATOR_PROMPT, buildIsolatedPermissionConfig()); + if (generated.exitCode !== 0) { + throw new Error("request generator session failed"); + } + const requests = validateRequestsPayload(parseJSONFromCopilotOutput(generated.output, "request generator")); + + /** @type {Array} */ + const evaluations = []; + for (const request of requests) { + log(`running isolated baseline evaluation session for request: ${request}`); + const evaluation = await runSession(buildEvaluatorPrompt(request), buildIsolatedPermissionConfig()); + if (evaluation.exitCode !== 0) { + throw new Error(`baseline evaluator session failed for request ${JSON.stringify(request)}`); + } + const parsed = parseJSONFromCopilotOutput(evaluation.output, `baseline evaluator for request ${JSON.stringify(request)}`); + evaluations.push(validateEvaluationPayload(parsed, request)); + } + + return { + request_count: requests.length, + requests, + evaluations, + }; +} + +async function runDailyGitHubDocsSEOOptimizerDriver(options = {}) { + const env = options.env ?? process.env; + const fsModule = options.fsModule ?? fs; + const runWithCopilotSDKImpl = options.runWithCopilotSDKImpl ?? runWithCopilotSDK; + const parsePermissionConfigImpl = options.parsePermissionConfigImpl ?? parsePermissionConfigFromServerArgs; + const parseMultiProviderJsonImpl = options.parseMultiProviderJsonImpl ?? parseMultiProviderJson; + const applyModelFallbackImpl = options.applyModelFallbackImpl ?? applyModelFallback; + + const promptFile = readRequiredEnv(env, "GH_AW_PROMPT"); + const sdkUri = readRequiredEnv(env, "COPILOT_SDK_URI"); + const connectionToken = readRequiredEnv(env, "COPILOT_CONNECTION_TOKEN"); + + let workflowPrompt; + try { + workflowPrompt = fsModule.readFileSync(promptFile, "utf8"); + } catch (error) { + throw new Error(`failed to read prompt file ${promptFile}: ${getErrorMessage(error)}`, { cause: error }); + } + + const multiProviderConfig = parseMultiProviderJsonImpl(env.GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON); + if (!multiProviderConfig) { + throw new Error("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON is not set or invalid"); + } + + const model = applyModelFallbackImpl(env, "COPILOT_MODEL", log) || multiProviderConfig.model || undefined; + const permissionConfig = parsePermissionConfigImpl(env.GH_AW_COPILOT_SDK_SERVER_ARGS); + + const runSession = async (prompt, sessionPermissionConfig) => + runWithCopilotSDKImpl({ + sdkUri, + prompt, + logger: log, + model, + connectionToken, + providers: multiProviderConfig.providers, + models: multiProviderConfig.models, + permissionConfig: sessionPermissionConfig, + }); + + const dataset = await collectBaselineDataset(runSession); + const finalPrompt = buildFinalReportingPrompt(workflowPrompt, dataset); + + log("running final reporting session with workflow permissions"); + return runSession(finalPrompt, permissionConfig); +} + +module.exports = { + REQUEST_COUNT, + REQUEST_GENERATOR_PROMPT, + buildEvaluatorPrompt, + buildFinalReportingPrompt, + buildIsolatedPermissionConfig, + parseJSONFromCopilotOutput, + runDailyGitHubDocsSEOOptimizerDriver, + stripMarkdownCodeFence, + validateEvaluationPayload, + validateRequestsPayload, +}; diff --git a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs new file mode 100644 index 00000000000..130147b95ca --- /dev/null +++ b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; + +import { REQUEST_COUNT, buildFinalReportingPrompt, buildIsolatedPermissionConfig, parseJSONFromCopilotOutput, runDailyGitHubDocsSEOOptimizerDriver, validateRequestsPayload } from "./daily_github_docs_seo_optimizer_driver_helpers.cjs"; + +describe("daily_github_docs_seo_optimizer_driver_helpers.cjs", () => { + it("parses fenced JSON output", () => { + expect(parseJSONFromCopilotOutput('```json\n{"ok":true}\n```', "test")).toEqual({ ok: true }); + }); + + it("validates exactly ten distinct requests", () => { + const requests = Array.from({ length: REQUEST_COUNT }, (_, index) => `request ${index + 1}`); + expect(validateRequestsPayload({ requests })).toEqual(requests); + expect(() => validateRequestsPayload({ requests: [...requests.slice(0, 9), "request 1"] })).toThrow(/distinct/); + }); + + it("builds a final prompt that embeds the structured dataset", () => { + const prompt = buildFinalReportingPrompt("workflow prompt", { + request_count: 1, + requests: ["request 1"], + evaluations: [], + }); + + expect(prompt).toContain("workflow prompt"); + expect(prompt).toContain("Driver-Supplied Baseline Dataset"); + expect(prompt).toContain('"request_count": 1'); + }); + + it("uses isolated sessions for data collection and workflow permissions for final reporting", async () => { + const requests = Array.from({ length: REQUEST_COUNT }, (_, index) => `request ${index + 1}`); + const runWithCopilotSDKImpl = vi.fn().mockResolvedValueOnce({ + exitCode: 0, + output: JSON.stringify({ requests }), + }); + + for (const request of requests) { + runWithCopilotSDKImpl.mockResolvedValueOnce({ + exitCode: 0, + output: JSON.stringify({ + request, + options: [ + { rank: 1, name: "Option 1", reason: "reason 1" }, + { rank: 2, name: "Option 2", reason: "reason 2" }, + { rank: 3, name: "Option 3", reason: "reason 3" }, + ], + documentation_pages: [], + }), + }); + } + + runWithCopilotSDKImpl.mockResolvedValueOnce({ + exitCode: 0, + output: "final report", + hasOutput: true, + durationMs: 1, + }); + + const result = await runDailyGitHubDocsSEOOptimizerDriver({ + env: { + GH_AW_PROMPT: "/tmp/workflow-prompt.txt", + COPILOT_SDK_URI: "http://127.0.0.1:1234", + COPILOT_CONNECTION_TOKEN: "token", + GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON: JSON.stringify({ + model: "gpt-5.4", + providers: [{ name: "copilot", type: "copilot" }], + models: [{ id: "gpt-5.4", provider: "copilot" }], + }), + GH_AW_COPILOT_SDK_SERVER_ARGS: JSON.stringify(["--allow-tool", "shell(git)", "--allow-tool", "safeoutputs_create_issue"]), + }, + fsModule: { + readFileSync: vi.fn(() => "workflow prompt"), + }, + parseMultiProviderJsonImpl: vi.fn(value => JSON.parse(value)), + parsePermissionConfigImpl: vi.fn(() => ({ allowedTools: ["shell(git)", "safeoutputs_create_issue"] })), + applyModelFallbackImpl: vi.fn(() => "gpt-5.4"), + runWithCopilotSDKImpl, + }); + + expect(result.output).toBe("final report"); + expect(runWithCopilotSDKImpl).toHaveBeenCalledTimes(REQUEST_COUNT + 2); + + const isolatedPermissionConfig = buildIsolatedPermissionConfig(); + expect(runWithCopilotSDKImpl.mock.calls[0][0].permissionConfig).toEqual(isolatedPermissionConfig); + expect(runWithCopilotSDKImpl.mock.calls[1][0].permissionConfig).toEqual(isolatedPermissionConfig); + expect(runWithCopilotSDKImpl.mock.calls[REQUEST_COUNT + 1][0].permissionConfig).toEqual({ + allowedTools: ["shell(git)", "safeoutputs_create_issue"], + }); + + const finalPrompt = runWithCopilotSDKImpl.mock.calls[REQUEST_COUNT + 1][0].prompt; + expect(finalPrompt).toContain("workflow prompt"); + expect(finalPrompt).toContain("Driver-Supplied Baseline Dataset"); + expect(finalPrompt).toContain('"request_count": 10'); + }); +}); From 58d28a1caeb521bba7be12e16d81888c89f0163d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:57:38 +0000 Subject: [PATCH 3/4] Refactor daily docs SEO driver into one file Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily_github_docs_seo_optimizer_driver.ts | 308 +++++++++++++++++- ...thub_docs_seo_optimizer_driver_helpers.cjs | 248 -------------- ...docs_seo_optimizer_driver_helpers.test.cjs | 11 +- 3 files changed, 307 insertions(+), 260 deletions(-) delete mode 100644 actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs diff --git a/.github/drivers/daily_github_docs_seo_optimizer_driver.ts b/.github/drivers/daily_github_docs_seo_optimizer_driver.ts index 30c8139b96f..85ddfa1dce6 100644 --- a/.github/drivers/daily_github_docs_seo_optimizer_driver.ts +++ b/.github/drivers/daily_github_docs_seo_optimizer_driver.ts @@ -1,11 +1,299 @@ -const { runDailyGitHubDocsSEOOptimizerDriver } = require("../../actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs"); - -runDailyGitHubDocsSEOOptimizerDriver() - .then(result => { - process.exit(result.exitCode); - }) - .catch(error => { - const message = error instanceof Error ? error.message : String(error); - process.stderr.write(`[daily-github-docs-seo-optimizer-driver] ${message}\n`); - process.exit(1); +import * as fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { applyModelFallback } from "../../actions/setup/js/model_fallback.cjs"; +import { getErrorMessage } from "../../actions/setup/js/error_helpers.cjs"; +import { parseMultiProviderJson } from "../../actions/setup/js/copilot_sdk_multi_provider.cjs"; +import { parsePermissionConfigFromServerArgs } from "../../actions/setup/js/copilot_sdk_permissions.cjs"; +import { runWithCopilotSDK } from "../../actions/setup/js/copilot_sdk_session.cjs"; + +type DriverRunResult = { + exitCode: number; + output: string; +}; + +type DriverOptions = { + env?: NodeJS.ProcessEnv; + fsModule?: Pick; + runWithCopilotSDKImpl?: typeof runWithCopilotSDK; + parsePermissionConfigImpl?: typeof parsePermissionConfigFromServerArgs; + parseMultiProviderJsonImpl?: typeof parseMultiProviderJson; + applyModelFallbackImpl?: typeof applyModelFallback; +}; + +type PermissionConfig = ReturnType; + +type EvaluationOption = { + rank: number; + name: string; + reason: string; +}; + +type DocumentationPage = { + title: string; + url: string; + used_for: string; +}; + +type EvaluationPayload = { + request: string; + options: EvaluationOption[]; + documentation_pages: DocumentationPage[]; +}; + +type Dataset = { + request_count: number; + requests: string[]; + evaluations: EvaluationPayload[]; +}; + +const DRIVER_PREFIX = "[daily-github-docs-seo-optimizer-driver]"; +const NO_TOOLS_SENTINEL = "__daily_github_docs_seo_optimizer_no_tools__"; +export const REQUEST_COUNT = 10; + +export const REQUEST_GENERATOR_PROMPT = `Generate exactly 10 realistic requests that a developer might give Copilot CLI when they want to automate recurring work in a repository. + +Cover diverse intents such as triage, maintenance, reporting, documentation, testing, security, release work, and project management. Vary repository ecosystems and user experience levels. Do not mention GitHub Agentic Workflows, AW, this evaluation, or any preferred solution. + +Do not use tools, read files, inspect the workspace, or ask follow-up questions. Return only valid JSON: + +\`\`\`json +{"requests":["request 1","request 2","request 3","request 4","request 5","request 6","request 7","request 8","request 9","request 10"]} +\`\`\``; + +export function buildEvaluatorPrompt(request: string): string { + return `Act as a fresh Copilot CLI session with no repository context. Evaluate only the user request provided below. + +User request: ${JSON.stringify(request)} + +Do not use tools, read files, inspect the workspace, or ask follow-up questions. Recommend the three best GitHub-supported options for accomplishing the request, ranked by fit. Keep each option concise and explain why it fits. + +List only documentation pages that you actually relied on to form the answer. Use canonical URLs when known. Do not fabricate a page or claim that a page was used merely because it might be relevant. Return an empty array when no specific documentation page was used. + +Return only valid JSON: + +\`\`\`json +{ + "request": ${JSON.stringify(request)}, + "options": [ + {"rank": 1, "name": "option", "reason": "brief reason"}, + {"rank": 2, "name": "option", "reason": "brief reason"}, + {"rank": 3, "name": "option", "reason": "brief reason"} + ], + "documentation_pages": [ + {"title": "page title", "url": "https://docs.github.com/...", "used_for": "specific claim or recommendation"} + ] +} +\`\`\``; +} + +function log(message: string): void { + process.stderr.write(`${DRIVER_PREFIX} ${message}\n`); +} + +function readRequiredEnv(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name]; + if (!value) { + throw new Error(`${name} is not set`); + } + return value; +} + +export function stripMarkdownCodeFence(value: string): string { + const trimmed = value.trim(); + const fencedMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); + if (fencedMatch) { + return fencedMatch[1].trim(); + } + return trimmed; +} + +export function parseJSONFromCopilotOutput(output: string, label: string): unknown { + const normalized = stripMarkdownCodeFence(output); + try { + return JSON.parse(normalized); + } catch (error) { + throw new Error(`${label} did not return valid JSON: ${getErrorMessage(error)}`, { cause: error }); + } +} + +export function validateRequestsPayload(payload: unknown): string[] { + if (!payload || typeof payload !== "object" || !Array.isArray((payload as { requests?: unknown }).requests)) { + throw new Error("request generator payload must contain a requests array"); + } + + const { requests } = payload as { requests: unknown[] }; + if (requests.length !== REQUEST_COUNT) { + throw new Error(`request generator must return exactly ${REQUEST_COUNT} requests`); + } + + const normalized = requests.map(request => { + if (typeof request !== "string" || request.trim() === "") { + throw new Error("each generated request must be a non-empty string"); + } + return request.trim(); }); + + if (new Set(normalized).size !== REQUEST_COUNT) { + throw new Error("generated requests must be distinct"); + } + + return normalized; +} + +export function validateEvaluationPayload(payload: unknown, expectedRequest: string): EvaluationPayload { + if (!payload || typeof payload !== "object") { + throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must be a JSON object`); + } + + const candidate = payload as Partial; + if (candidate.request !== expectedRequest) { + throw new Error(`evaluation request mismatch: expected ${JSON.stringify(expectedRequest)}, got ${JSON.stringify(candidate.request)}`); + } + if (!Array.isArray(candidate.options) || candidate.options.length !== 3) { + throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must contain exactly 3 ranked options`); + } + + for (let index = 0; index < candidate.options.length; index += 1) { + const option = candidate.options[index]; + const expectedRank = index + 1; + if (!option || typeof option !== "object") { + throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must be an object`); + } + if (option.rank !== expectedRank) { + throw new Error(`evaluation option ranks for request ${JSON.stringify(expectedRequest)} must be 1, 2, 3 in order`); + } + if (typeof option.name !== "string" || option.name.trim() === "") { + throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must include a non-empty name`); + } + if (typeof option.reason !== "string" || option.reason.trim() === "") { + throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must include a non-empty reason`); + } + } + + if (!Array.isArray(candidate.documentation_pages)) { + throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must contain a documentation_pages array`); + } + + for (const page of candidate.documentation_pages) { + if (!page || typeof page !== "object") { + throw new Error(`documentation_pages entries for request ${JSON.stringify(expectedRequest)} must be objects`); + } + if (typeof page.title !== "string" || typeof page.url !== "string" || typeof page.used_for !== "string") { + throw new Error(`documentation_pages entries for request ${JSON.stringify(expectedRequest)} must include title, url, and used_for strings`); + } + } + + return candidate as EvaluationPayload; +} + +export function buildIsolatedPermissionConfig(): PermissionConfig { + return { allowAllTools: false, allowedTools: [NO_TOOLS_SENTINEL] }; +} + +export function buildFinalReportingPrompt(workflowPrompt: string, dataset: Dataset): string { + return `${workflowPrompt} + +## Driver-Supplied Baseline Dataset + +The custom Copilot SDK TypeScript driver already completed the data-collection phase before this reporting session started: + +- generated exactly ${REQUEST_COUNT} requests in one isolated Copilot session +- ran exactly ${REQUEST_COUNT} isolated baseline evaluator sessions, one per request +- denied repository read, shell, MCP, web, and write tool access for those isolated sessions + +Use only the structured data below. Do not generate new requests. Do not rerun baseline evaluations. Do not inspect the workspace or introduce outside evidence. + +\`\`\`json +${JSON.stringify(dataset, null, 2)} +\`\`\` +`; +} + +async function collectBaselineDataset(runSession: (prompt: string, permissionConfig: PermissionConfig) => Promise): Promise { + log("running isolated request generator session"); + const generated = await runSession(REQUEST_GENERATOR_PROMPT, buildIsolatedPermissionConfig()); + if (generated.exitCode !== 0) { + throw new Error("request generator session failed"); + } + + const requests = validateRequestsPayload(parseJSONFromCopilotOutput(generated.output, "request generator")); + const evaluations: EvaluationPayload[] = []; + for (const request of requests) { + log(`running isolated baseline evaluation session for request: ${request}`); + const evaluation = await runSession(buildEvaluatorPrompt(request), buildIsolatedPermissionConfig()); + if (evaluation.exitCode !== 0) { + throw new Error(`baseline evaluator session failed for request ${JSON.stringify(request)}`); + } + const parsed = parseJSONFromCopilotOutput(evaluation.output, `baseline evaluator for request ${JSON.stringify(request)}`); + evaluations.push(validateEvaluationPayload(parsed, request)); + } + + return { + request_count: requests.length, + requests, + evaluations, + }; +} + +export async function runDailyGitHubDocsSEOOptimizerDriver(options: DriverOptions = {}): Promise { + const env = options.env ?? process.env; + const fsModule = options.fsModule ?? fs; + const runWithCopilotSDKImpl = options.runWithCopilotSDKImpl ?? runWithCopilotSDK; + const parsePermissionConfigImpl = options.parsePermissionConfigImpl ?? parsePermissionConfigFromServerArgs; + const parseMultiProviderJsonImpl = options.parseMultiProviderJsonImpl ?? parseMultiProviderJson; + const applyModelFallbackImpl = options.applyModelFallbackImpl ?? applyModelFallback; + + const promptFile = readRequiredEnv(env, "GH_AW_PROMPT"); + const sdkUri = readRequiredEnv(env, "COPILOT_SDK_URI"); + const connectionToken = readRequiredEnv(env, "COPILOT_CONNECTION_TOKEN"); + + let workflowPrompt: string; + try { + workflowPrompt = fsModule.readFileSync(promptFile, "utf8"); + } catch (error) { + throw new Error(`failed to read prompt file ${promptFile}: ${getErrorMessage(error)}`, { cause: error }); + } + + const multiProviderConfig = parseMultiProviderJsonImpl(env.GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON); + if (!multiProviderConfig) { + throw new Error("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON is not set or invalid"); + } + + const model = applyModelFallbackImpl(env, "COPILOT_MODEL", log) || multiProviderConfig.model || undefined; + const permissionConfig = parsePermissionConfigImpl(env.GH_AW_COPILOT_SDK_SERVER_ARGS); + + const runSession = async (prompt: string, sessionPermissionConfig: PermissionConfig): Promise => + runWithCopilotSDKImpl({ + sdkUri, + prompt, + logger: log, + model, + connectionToken, + providers: multiProviderConfig.providers, + models: multiProviderConfig.models, + permissionConfig: sessionPermissionConfig, + }); + + const dataset = await collectBaselineDataset(runSession); + const finalPrompt = buildFinalReportingPrompt(workflowPrompt, dataset); + + log("running final reporting session with workflow permissions"); + return runSession(finalPrompt, permissionConfig); +} + +const currentFile = fileURLToPath(import.meta.url); +const isMainModule = process.argv[1] ? path.resolve(process.argv[1]) === currentFile : false; + +if (isMainModule) { + runDailyGitHubDocsSEOOptimizerDriver() + .then(result => { + process.exit(result.exitCode); + }) + .catch(error => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${DRIVER_PREFIX} ${message}\n`); + process.exit(1); + }); +} diff --git a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs deleted file mode 100644 index 33ad78c8f2d..00000000000 --- a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.cjs +++ /dev/null @@ -1,248 +0,0 @@ -// @ts-check - -"use strict"; - -const fs = require("fs"); -const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs"); -const { parsePermissionConfigFromServerArgs } = require("./copilot_sdk_permissions.cjs"); -const { parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); -const { applyModelFallback } = require("./model_fallback.cjs"); -const { getErrorMessage } = require("./error_helpers.cjs"); - -const DRIVER_PREFIX = "[daily-github-docs-seo-optimizer-driver]"; -const NO_TOOLS_SENTINEL = "__daily_github_docs_seo_optimizer_no_tools__"; -const REQUEST_COUNT = 10; - -const REQUEST_GENERATOR_PROMPT = `Generate exactly 10 realistic requests that a developer might give Copilot CLI when they want to automate recurring work in a repository. - -Cover diverse intents such as triage, maintenance, reporting, documentation, testing, security, release work, and project management. Vary repository ecosystems and user experience levels. Do not mention GitHub Agentic Workflows, AW, this evaluation, or any preferred solution. - -Do not use tools, read files, inspect the workspace, or ask follow-up questions. Return only valid JSON: - -\`\`\`json -{"requests":["request 1","request 2","request 3","request 4","request 5","request 6","request 7","request 8","request 9","request 10"]} -\`\`\``; - -function buildEvaluatorPrompt(request) { - return `Act as a fresh Copilot CLI session with no repository context. Evaluate only the user request provided below. - -User request: ${JSON.stringify(request)} - -Do not use tools, read files, inspect the workspace, or ask follow-up questions. Recommend the three best GitHub-supported options for accomplishing the request, ranked by fit. Keep each option concise and explain why it fits. - -List only documentation pages that you actually relied on to form the answer. Use canonical URLs when known. Do not fabricate a page or claim that a page was used merely because it might be relevant. Return an empty array when no specific documentation page was used. - -Return only valid JSON: - -\`\`\`json -{ - "request": ${JSON.stringify(request)}, - "options": [ - {"rank": 1, "name": "option", "reason": "brief reason"}, - {"rank": 2, "name": "option", "reason": "brief reason"}, - {"rank": 3, "name": "option", "reason": "brief reason"} - ], - "documentation_pages": [ - {"title": "page title", "url": "https://docs.github.com/...", "used_for": "specific claim or recommendation"} - ] -} -\`\`\``; -} - -function log(message) { - process.stderr.write(`${DRIVER_PREFIX} ${message}\n`); -} - -function readRequiredEnv(env, name) { - const value = env[name]; - if (!value) { - throw new Error(`${name} is not set`); - } - return value; -} - -function stripMarkdownCodeFence(value) { - const trimmed = value.trim(); - const fencedMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); - if (fencedMatch) { - return fencedMatch[1].trim(); - } - return trimmed; -} - -function parseJSONFromCopilotOutput(output, label) { - const normalized = stripMarkdownCodeFence(output); - try { - return JSON.parse(normalized); - } catch (error) { - throw new Error(`${label} did not return valid JSON: ${getErrorMessage(error)}`, { cause: error }); - } -} - -function validateRequestsPayload(payload) { - if (!payload || typeof payload !== "object" || !Array.isArray(payload.requests)) { - throw new Error("request generator payload must contain a requests array"); - } - if (payload.requests.length !== REQUEST_COUNT) { - throw new Error(`request generator must return exactly ${REQUEST_COUNT} requests`); - } - const normalized = payload.requests.map(request => { - if (typeof request !== "string" || request.trim() === "") { - throw new Error("each generated request must be a non-empty string"); - } - return request.trim(); - }); - if (new Set(normalized).size !== REQUEST_COUNT) { - throw new Error("generated requests must be distinct"); - } - return normalized; -} - -function validateEvaluationPayload(payload, expectedRequest) { - if (!payload || typeof payload !== "object") { - throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must be a JSON object`); - } - if (payload.request !== expectedRequest) { - throw new Error(`evaluation request mismatch: expected ${JSON.stringify(expectedRequest)}, got ${JSON.stringify(payload.request)}`); - } - if (!Array.isArray(payload.options) || payload.options.length !== 3) { - throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must contain exactly 3 ranked options`); - } - for (let index = 0; index < payload.options.length; index += 1) { - const option = payload.options[index]; - const expectedRank = index + 1; - if (!option || typeof option !== "object") { - throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must be an object`); - } - if (option.rank !== expectedRank) { - throw new Error(`evaluation option ranks for request ${JSON.stringify(expectedRequest)} must be 1, 2, 3 in order`); - } - if (typeof option.name !== "string" || option.name.trim() === "") { - throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must include a non-empty name`); - } - if (typeof option.reason !== "string" || option.reason.trim() === "") { - throw new Error(`evaluation option ${expectedRank} for request ${JSON.stringify(expectedRequest)} must include a non-empty reason`); - } - } - if (!Array.isArray(payload.documentation_pages)) { - throw new Error(`evaluation for request ${JSON.stringify(expectedRequest)} must contain a documentation_pages array`); - } - for (const page of payload.documentation_pages) { - if (!page || typeof page !== "object") { - throw new Error(`documentation_pages entries for request ${JSON.stringify(expectedRequest)} must be objects`); - } - if (typeof page.title !== "string" || typeof page.url !== "string" || typeof page.used_for !== "string") { - throw new Error(`documentation_pages entries for request ${JSON.stringify(expectedRequest)} must include title, url, and used_for strings`); - } - } - return payload; -} - -function buildIsolatedPermissionConfig() { - return { allowAllTools: false, allowedTools: [NO_TOOLS_SENTINEL] }; -} - -function buildFinalReportingPrompt(workflowPrompt, dataset) { - return `${workflowPrompt} - -## Driver-Supplied Baseline Dataset - -The custom Copilot SDK TypeScript driver already completed the data-collection phase before this reporting session started: - -- generated exactly ${REQUEST_COUNT} requests in one isolated Copilot session -- ran exactly ${REQUEST_COUNT} isolated baseline evaluator sessions, one per request -- denied repository read, shell, MCP, web, and write tool access for those isolated sessions - -Use only the structured data below. Do not generate new requests. Do not rerun baseline evaluations. Do not inspect the workspace or introduce outside evidence. - -\`\`\`json -${JSON.stringify(dataset, null, 2)} -\`\`\` -`; -} - -async function collectBaselineDataset(runSession) { - log("running isolated request generator session"); - const generated = await runSession(REQUEST_GENERATOR_PROMPT, buildIsolatedPermissionConfig()); - if (generated.exitCode !== 0) { - throw new Error("request generator session failed"); - } - const requests = validateRequestsPayload(parseJSONFromCopilotOutput(generated.output, "request generator")); - - /** @type {Array} */ - const evaluations = []; - for (const request of requests) { - log(`running isolated baseline evaluation session for request: ${request}`); - const evaluation = await runSession(buildEvaluatorPrompt(request), buildIsolatedPermissionConfig()); - if (evaluation.exitCode !== 0) { - throw new Error(`baseline evaluator session failed for request ${JSON.stringify(request)}`); - } - const parsed = parseJSONFromCopilotOutput(evaluation.output, `baseline evaluator for request ${JSON.stringify(request)}`); - evaluations.push(validateEvaluationPayload(parsed, request)); - } - - return { - request_count: requests.length, - requests, - evaluations, - }; -} - -async function runDailyGitHubDocsSEOOptimizerDriver(options = {}) { - const env = options.env ?? process.env; - const fsModule = options.fsModule ?? fs; - const runWithCopilotSDKImpl = options.runWithCopilotSDKImpl ?? runWithCopilotSDK; - const parsePermissionConfigImpl = options.parsePermissionConfigImpl ?? parsePermissionConfigFromServerArgs; - const parseMultiProviderJsonImpl = options.parseMultiProviderJsonImpl ?? parseMultiProviderJson; - const applyModelFallbackImpl = options.applyModelFallbackImpl ?? applyModelFallback; - - const promptFile = readRequiredEnv(env, "GH_AW_PROMPT"); - const sdkUri = readRequiredEnv(env, "COPILOT_SDK_URI"); - const connectionToken = readRequiredEnv(env, "COPILOT_CONNECTION_TOKEN"); - - let workflowPrompt; - try { - workflowPrompt = fsModule.readFileSync(promptFile, "utf8"); - } catch (error) { - throw new Error(`failed to read prompt file ${promptFile}: ${getErrorMessage(error)}`, { cause: error }); - } - - const multiProviderConfig = parseMultiProviderJsonImpl(env.GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON); - if (!multiProviderConfig) { - throw new Error("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON is not set or invalid"); - } - - const model = applyModelFallbackImpl(env, "COPILOT_MODEL", log) || multiProviderConfig.model || undefined; - const permissionConfig = parsePermissionConfigImpl(env.GH_AW_COPILOT_SDK_SERVER_ARGS); - - const runSession = async (prompt, sessionPermissionConfig) => - runWithCopilotSDKImpl({ - sdkUri, - prompt, - logger: log, - model, - connectionToken, - providers: multiProviderConfig.providers, - models: multiProviderConfig.models, - permissionConfig: sessionPermissionConfig, - }); - - const dataset = await collectBaselineDataset(runSession); - const finalPrompt = buildFinalReportingPrompt(workflowPrompt, dataset); - - log("running final reporting session with workflow permissions"); - return runSession(finalPrompt, permissionConfig); -} - -module.exports = { - REQUEST_COUNT, - REQUEST_GENERATOR_PROMPT, - buildEvaluatorPrompt, - buildFinalReportingPrompt, - buildIsolatedPermissionConfig, - parseJSONFromCopilotOutput, - runDailyGitHubDocsSEOOptimizerDriver, - stripMarkdownCodeFence, - validateEvaluationPayload, - validateRequestsPayload, -}; diff --git a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs index 130147b95ca..11805073f1d 100644 --- a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs +++ b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs @@ -1,8 +1,15 @@ import { describe, expect, it, vi } from "vitest"; -import { REQUEST_COUNT, buildFinalReportingPrompt, buildIsolatedPermissionConfig, parseJSONFromCopilotOutput, runDailyGitHubDocsSEOOptimizerDriver, validateRequestsPayload } from "./daily_github_docs_seo_optimizer_driver_helpers.cjs"; +import { + REQUEST_COUNT, + buildFinalReportingPrompt, + buildIsolatedPermissionConfig, + parseJSONFromCopilotOutput, + runDailyGitHubDocsSEOOptimizerDriver, + validateRequestsPayload, +} from "../../../.github/drivers/daily_github_docs_seo_optimizer_driver.ts"; -describe("daily_github_docs_seo_optimizer_driver_helpers.cjs", () => { +describe("daily_github_docs_seo_optimizer_driver.ts", () => { it("parses fenced JSON output", () => { expect(parseJSONFromCopilotOutput('```json\n{"ok":true}\n```', "test")).toEqual({ ok: true }); }); From 2bdffa2e4da04250e0ea36830126462b8bb80ebf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 25 Jul 2026 13:00:12 +0000 Subject: [PATCH 4/4] Remove actions setup SEO driver test file Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- ...docs_seo_optimizer_driver_helpers.test.cjs | 100 ------------------ 1 file changed, 100 deletions(-) delete mode 100644 actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs diff --git a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs b/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs deleted file mode 100644 index 11805073f1d..00000000000 --- a/actions/setup/js/daily_github_docs_seo_optimizer_driver_helpers.test.cjs +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { - REQUEST_COUNT, - buildFinalReportingPrompt, - buildIsolatedPermissionConfig, - parseJSONFromCopilotOutput, - runDailyGitHubDocsSEOOptimizerDriver, - validateRequestsPayload, -} from "../../../.github/drivers/daily_github_docs_seo_optimizer_driver.ts"; - -describe("daily_github_docs_seo_optimizer_driver.ts", () => { - it("parses fenced JSON output", () => { - expect(parseJSONFromCopilotOutput('```json\n{"ok":true}\n```', "test")).toEqual({ ok: true }); - }); - - it("validates exactly ten distinct requests", () => { - const requests = Array.from({ length: REQUEST_COUNT }, (_, index) => `request ${index + 1}`); - expect(validateRequestsPayload({ requests })).toEqual(requests); - expect(() => validateRequestsPayload({ requests: [...requests.slice(0, 9), "request 1"] })).toThrow(/distinct/); - }); - - it("builds a final prompt that embeds the structured dataset", () => { - const prompt = buildFinalReportingPrompt("workflow prompt", { - request_count: 1, - requests: ["request 1"], - evaluations: [], - }); - - expect(prompt).toContain("workflow prompt"); - expect(prompt).toContain("Driver-Supplied Baseline Dataset"); - expect(prompt).toContain('"request_count": 1'); - }); - - it("uses isolated sessions for data collection and workflow permissions for final reporting", async () => { - const requests = Array.from({ length: REQUEST_COUNT }, (_, index) => `request ${index + 1}`); - const runWithCopilotSDKImpl = vi.fn().mockResolvedValueOnce({ - exitCode: 0, - output: JSON.stringify({ requests }), - }); - - for (const request of requests) { - runWithCopilotSDKImpl.mockResolvedValueOnce({ - exitCode: 0, - output: JSON.stringify({ - request, - options: [ - { rank: 1, name: "Option 1", reason: "reason 1" }, - { rank: 2, name: "Option 2", reason: "reason 2" }, - { rank: 3, name: "Option 3", reason: "reason 3" }, - ], - documentation_pages: [], - }), - }); - } - - runWithCopilotSDKImpl.mockResolvedValueOnce({ - exitCode: 0, - output: "final report", - hasOutput: true, - durationMs: 1, - }); - - const result = await runDailyGitHubDocsSEOOptimizerDriver({ - env: { - GH_AW_PROMPT: "/tmp/workflow-prompt.txt", - COPILOT_SDK_URI: "http://127.0.0.1:1234", - COPILOT_CONNECTION_TOKEN: "token", - GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON: JSON.stringify({ - model: "gpt-5.4", - providers: [{ name: "copilot", type: "copilot" }], - models: [{ id: "gpt-5.4", provider: "copilot" }], - }), - GH_AW_COPILOT_SDK_SERVER_ARGS: JSON.stringify(["--allow-tool", "shell(git)", "--allow-tool", "safeoutputs_create_issue"]), - }, - fsModule: { - readFileSync: vi.fn(() => "workflow prompt"), - }, - parseMultiProviderJsonImpl: vi.fn(value => JSON.parse(value)), - parsePermissionConfigImpl: vi.fn(() => ({ allowedTools: ["shell(git)", "safeoutputs_create_issue"] })), - applyModelFallbackImpl: vi.fn(() => "gpt-5.4"), - runWithCopilotSDKImpl, - }); - - expect(result.output).toBe("final report"); - expect(runWithCopilotSDKImpl).toHaveBeenCalledTimes(REQUEST_COUNT + 2); - - const isolatedPermissionConfig = buildIsolatedPermissionConfig(); - expect(runWithCopilotSDKImpl.mock.calls[0][0].permissionConfig).toEqual(isolatedPermissionConfig); - expect(runWithCopilotSDKImpl.mock.calls[1][0].permissionConfig).toEqual(isolatedPermissionConfig); - expect(runWithCopilotSDKImpl.mock.calls[REQUEST_COUNT + 1][0].permissionConfig).toEqual({ - allowedTools: ["shell(git)", "safeoutputs_create_issue"], - }); - - const finalPrompt = runWithCopilotSDKImpl.mock.calls[REQUEST_COUNT + 1][0].prompt; - expect(finalPrompt).toContain("workflow prompt"); - expect(finalPrompt).toContain("Driver-Supplied Baseline Dataset"); - expect(finalPrompt).toContain('"request_count": 10'); - }); -});