engineer-bot.yml

# Engineer Bot — author.
#
# A maintainer labels an ISSUE with `engineer-bot`; the bot addresses it and opens
# a PR. The author FLOW is chosen from the issue's TYPE (a GitHub org-level Issue
# Type): `Bug` ⇒ bug-fix (reproduce with a failing test, then fix); any other type
# / none ⇒ the repo's .bot/config.yaml `flow:` default (also bug-fix). Only bug-fix
# is selectable because this repo's .bot/prompts/engineer/ prompts are written
# exclusively for it; a task-flavored flow would need matching prompts first. Set
# the Type BEFORE `engineer-bot` — it's read live when the run starts. Driven by
# the selected flow + this repo's .bot/ (config + prompts).
#
# The engineer builds and runs THIS repo's tests, so it needs the connector's
# deps installed (via setup-poetry) in addition to the engine. Shared setup
# (mint tokens + Node + install the pinned engine, PAT-free) is factored into the
# local ./.github/actions/bot-prelude composite; this file owns checkout,
# setup-poetry, and the author/publish/comment tail. The engine is installed in
# REF mode pinned to a SHA and authenticated PAT-free by an engine-scoped App
# token (minted inside bot-prelude).
#
# SECURITY: the trigger is a label only maintainers can apply — issue BODIES are
# untrusted input, so the gate is who-can-label, not the content. The issue body
# is written to a file and inlined into the agent prompt; it never reaches a shell.
name: Engineer Bot

on:
  issues:
    types: [labeled]
  workflow_dispatch:
    inputs:
      issue_number:
        description: 'Issue number to fix'
        required: true
        type: string

permissions:
  contents: write        # push the fix branch
  pull-requests: write   # open the fix PR
  issues: write          # comment back on the tracking issue
  id-token: write        # JFrog OIDC exchange for the engine/SDK/CLI install

jobs:
  author:
    # Manual dispatch, OR the `engineer-bot` label was just applied. Applying a
    # label requires triage+ on the repo, so this is the maintainer-only gate.
    # The `sender.type != 'Bot'` guard is defense-in-depth (matching the followup
    # gate): a GitHub App with `issues: write` could apply the label without a
    # human, so the triage+ argument doesn't fully cover automation.
    if: >-
      github.event_name == 'workflow_dispatch'
      || (github.event.label.name == 'engineer-bot'
          && github.event.sender.type != 'Bot')
    environment: azure-prod   # DATABRICKS_HOST / DATABRICKS_TOKEN live here
    runs-on:
      group: databricks-protected-runner-group
      labels: [linux-ubuntu-latest]
    timeout-minutes: 45
    concurrency:
      # One author run per issue; a re-label while a run is in flight queues.
      group: engineer-bot-issue-${{ github.event.issue.number || inputs.issue_number }}
      cancel-in-progress: false
    steps:
      # Checkout FIRST (remote action) so the local `./` composites resolve.
      # publish sets its own authenticated push remote for the fix branch, so
      # persist-credentials:false is fine.
      - name: Checkout (default branch)
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          fetch-depth: 0
          persist-credentials: false

      # Python + connector deps via the repo's OWN setup-poetry action: it
      # configures poetry against the internal JFrog mirror (the protected runner
      # is egress-blocked from pypi.org), runs `poetry lock` (tolerating a stale
      # lock), and installs the connector + its deps. This is the maintained,
      # egress-safe way this repo installs itself — the agent then runs
      # `poetry run python -m pytest tests/unit` to self-verify its red→green fix.
      # (The engineer builds/runs the connector, so unlike the reviewer it needs
      # the deps, not just an interpreter.)
      - name: Setup Poetry + connector deps (for pytest self-verify)
        uses: ./.github/actions/setup-poetry
        with:
          python-version: '3.11'
          # Match code-coverage.yml's e2e job: install optional extras (notably
          # pyarrow, declared optional=true behind the `pyarrow` extra) so the
          # agent's REQUIRED live E2E repro test runs against the same runtime as
          # the e2e suite it mirrors. Without this, arrow-touching e2e tests
          # ModuleNotFoundError/skip instead of failing red-for-the-right-reason.
          install-args: "--all-extras"

      # setup-poetry runs `poetry lock` (to reconcile the lock with the internal
      # JFrog source it injects), which REWRITES tracked poetry.lock / pyproject.toml
      # in the working tree. The venv is already built, so the tree no longer needs
      # that churn — revert it, else publish's leftover safety-net sees these
      # bot-untouched dirty files and refuses to open the PR (they aren't in the
      # agent's touched_files). Keep ONLY the agent's edits in the tree.
      - name: Revert poetry lock/pyproject churn (keep only the agent's edits)
        # No `|| true`: `git checkout -- <tracked-file>` exits 0 for both the
        # clean and the reverted-churn cases, so a non-zero exit means the revert
        # genuinely failed (e.g. the files went untracked/missing) and the dirty
        # tree would otherwise resurface as a confusing publish-time error. Fail
        # loudly here instead of masking it.
        run: git checkout -- poetry.lock pyproject.toml

      # Shared prelude: mint the engineer-bot token (pushes the fix branch,
      # comments) + the engine-scoped token, set up Node, install the pinned
      # engine (PAT-free). `token` output is used by the steps below.
      - name: Bot prelude (tokens + Node + engine install)
        id: prelude
        uses: ./.github/actions/bot-prelude
        with:
          app-id: ${{ secrets.ENGINEER_BOT_APP_ID }}
          private-key: ${{ secrets.ENGINEER_BOT_APP_PRIVATE_KEY }}
          # Engine pin lives in ONE place: bot-prelude's `engine-ref` default.
          # Bump it there to move all four bots in lockstep; override here only
          # to run this workflow against a different engine commit.

      - name: Resolve issue + gather context
        id: ctx
        # SECURITY: the issue number is validated as digits-only before use; the
        # issue body + title are UNTRUSTED. The body goes to a file (never a
        # shell or env interpolation); the title is exported through $GITHUB_ENV
        # with a random heredoc delimiter (a title can contain newlines, or a
        # chosen delimiter, which would otherwise inject extra env vars). The
        # number/url are safe shapes and exported directly.
        env:
          GH_TOKEN: ${{ steps.prelude.outputs.token }}
          EVENT_NAME: ${{ github.event_name }}
          INPUT_ISSUE: ${{ inputs.issue_number }}
          EVENT_ISSUE: ${{ github.event.issue.number }}
          REPO: ${{ github.repository }}
        run: |
          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then RAW="$INPUT_ISSUE"; else RAW="$EVENT_ISSUE"; fi
          [[ "$RAW" =~ ^[0-9]+$ ]] || { echo "::error::Invalid issue number '$RAW'"; exit 1; }
          # `gh issue view --json` does NOT expose the issue TYPE field, so fetch
          # via the REST API (which carries `.type.name`). Body/title/url still
          # come from the same JSON. Issue Types are an ORG-level feature; a repo
          # without them configured returns `.type == null` (⇒ no --flow ⇒ the
          # engine uses the .bot/config.yaml `flow:` default).
          gh api "repos/$REPO/issues/$RAW" > "$RUNNER_TEMP/issue.json"
          # The REST issues endpoint resolves PR numbers too (PRs are issues in
          # the REST model), unlike `gh issue view <pr-number>` which errored.
          # Keep rejecting PR numbers cleanly: real issues have `.pull_request == null`.
          [ "$(jq -r '.pull_request // "null"' "$RUNNER_TEMP/issue.json")" = "null" ] \
            || { echo "::error::#$RAW is a pull request, not an issue"; exit 1; }
          jq -r '.body // ""' "$RUNNER_TEMP/issue.json" > "$RUNNER_TEMP/issue_body.txt"
          TITLE="$(jq -r '.title // ""' "$RUNNER_TEMP/issue.json")"
          URL="$(jq -r '.html_url // ""' "$RUNNER_TEMP/issue.json")"
          {
            echo "ISSUE_NUMBER=$RAW"
            echo "ISSUE_URL=$URL"
          } >> "$GITHUB_ENV"
          echo "issue_number=$RAW" >> "$GITHUB_OUTPUT"
          # Derive the author flow from the issue's TYPE (read LIVE here, so a type
          # set AFTER this fetch is not seen — set the type BEFORE `engineer-bot`).
          # The type→flow MAPPING is pinned (not "use the type name as the flow")
          # so an arbitrary custom org type can never select an engine flow:
          #   Bug ⇒ bug-fix (red→green TDD)
          # Only `bug-fix` is mapped: this repo's .bot/prompts/engineer/{system,user}.md
          # are written EXCLUSIVELY for the bug-fix flow ("reproduce the bug with a
          # failing test, then fix"). Mapping Task/Feature ⇒ task would run the
          # engine's single-pass `task` flow under a prompt that mandates red→green
          # bug reproduction — a flow/prompt contradiction. Until flow-aware prompts
          # exist, every run uses bug-fix.
          # Any other type or no type ⇒ emit nothing ⇒ the author step omits
          # --flow ⇒ the engine uses .bot/config.yaml `flow:` default (bug-fix).
          TYPE="$(jq -r '.type.name // ""' "$RUNNER_TEMP/issue.json")"
          case "$TYPE" in
            Bug) echo "flow=bug-fix" >> "$GITHUB_OUTPUT" ;;
          esac
          DELIM="GHEOF_$(date +%s%N)${RANDOM}"
          if printf '%s' "$TITLE" | grep -qF "$DELIM"; then
            echo "::error::title delimiter collision — refusing to write \$GITHUB_ENV"; exit 1
          fi
          { echo "ISSUE_TITLE<<$DELIM"; printf '%s\n' "$TITLE"; echo "$DELIM"; } >> "$GITHUB_ENV"

      - name: Run author
        id: author
        # Run from RUNNER_TEMP so the engine-rendered prompt's context file
        # (issue_body.txt, resolved against cwd) is read from there and the
        # checkout stays clean — publish's leftover check fails on any untracked
        # path in the repo. REPO_ROOT points the agent's working tree (and
        # the .bot/ lookup) at the checkout; ISSUE_NUMBER/TITLE/URL come from
        # $GITHUB_ENV (the bot config's author.env_tokens).
        #
        # The author FLOW is chosen from the issue TYPE by the ctx step above
        # (Bug ⇒ bug-fix; any other/none ⇒ config default). We pass --flow ONLY
        # when that step derived one: an empty --flow would be rejected by the CLI's
        # `choices`, and omitting it lets the engine fall back to the
        # .bot/config.yaml `flow:` default (no/other type, or type-set-late case).
        working-directory: ${{ runner.temp }}
        env:
          MODEL_ENDPOINT: https://${{ secrets.DATABRICKS_HOST }}/serving-endpoints/databricks-claude-opus-4-8/invocations
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
          # Live-warehouse connection env for the agent's REQUIRED E2E repro test
          # (see .bot/prompts/engineer/system.md — the bug-fix flow reproduces the
          # bug via a tests/e2e test against a real warehouse; a mocked unit test
          # alone cannot prove the live server behaves correctly). Mirrors the
          # e2e job in code-coverage.yml; the job already runs in `environment:
          # azure-prod`, so these secrets are in scope.
          DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
          DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
          DATABRICKS_CATALOG: peco
          DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
          REPO_ROOT: ${{ github.workspace }}
          RUNNER_TEMP: ${{ runner.temp }}
          FLOW: ${{ steps.ctx.outputs.flow }}
        run: |
          args=(
            --phase author
            --system-prompt "$GITHUB_WORKSPACE/.bot/prompts/engineer/system.md"
            --user-prompt "$GITHUB_WORKSPACE/.bot/prompts/engineer/user.md"
          )
          [ -n "$FLOW" ] && args+=(--flow "$FLOW")
          python -m databricks_bot_engine.engineer_bot.run "${args[@]}"

      - name: Open / update fix PR
        id: publish
        if: steps.author.outputs.outcome == 'success'
        # publish stages exactly the agent-touched files, commits as the bot,
        # pushes the fix branch, and opens/updates the PR. The push authenticates
        # via the App token embedded in the remote URL (checkout ran with
        # persist-credentials: false, so this is the only push credential).
        env:
          GH_TOKEN: ${{ steps.prelude.outputs.token }}
          GITHUB_REPOSITORY: ${{ github.repository }}
          REPO_ROOT: ${{ github.workspace }}
          RUNNER_TEMP: ${{ runner.temp }}
          ENGINEER_BOT_APP_ID: ${{ secrets.ENGINEER_BOT_APP_ID }}
          TOUCHED_FILES: ${{ steps.author.outputs.touched_files }}
          TOUCHED_COUNT: ${{ steps.author.outputs.touched_count }}
          SUMMARY: ${{ steps.author.outputs.summary }}
          RED_GREEN_TESTS: ${{ steps.author.outputs.red_green_tests }}
          PLAN: ${{ steps.author.outputs.plan }}
          OUT_OF_SCOPE: ${{ steps.author.outputs.out_of_scope }}
          # ISSUE_NUMBER / ISSUE_TITLE / ISSUE_URL come from $GITHUB_ENV.
        run: |
          git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
          python -m databricks_bot_engine.engineer_bot.publish --repo-dir "$GITHUB_WORKSPACE"

      - name: Comment outcome on issue
        # Always close the loop on the tracking issue, whatever happened. The
        # agent SUMMARY / reason is LLM-generated — write it to a file and pass
        # via --body-file so nothing reaches a shell.
        if: always() && steps.ctx.outputs.issue_number != ''
        env:
          GH_TOKEN: ${{ steps.prelude.outputs.token }}
          REPO: ${{ github.repository }}
          ISSUE: ${{ steps.ctx.outputs.issue_number }}
          OUTCOME: ${{ steps.author.outputs.outcome }}
          # `coverage_pr_url` is the engine publish module's PR-URL output key for
          # ALL flows (bug-fix included), not just coverage — the name is legacy.
          # engineer_bot.publish emits this key regardless of flow, so it is correct here.
          PR_URL: ${{ steps.publish.outputs.coverage_pr_url }}
          REASON: ${{ steps.author.outputs.reason }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          BODY="$RUNNER_TEMP/issue_comment.md"
          # $OUTCOME is the engine's emitted outcome (steps.author.outputs.outcome),
          # NOT the agent's internal enum. The engine maps the agent's
          # `no_change_needed` (what the prompt asks the agent to report) to the
          # `$GITHUB_OUTPUT` value `no_change` (run_author.py). So `no_change)` below
          # is the correct arm — do not "fix" it to `no_change_needed)`.
          case "$OUTCOME" in
            success)
              if [ -n "$PR_URL" ]; then
                printf '🤖 engineer-bot opened a fix PR: %s\n\nReview before merge.\n' "$PR_URL" > "$BODY"
              else
                printf '🤖 engineer-bot produced a fix but could **not** open the PR (push or PR creation failed). See the [workflow run](%s) for details.\n' "$RUN_URL" > "$BODY"
              fi ;;
            no_change)
              { printf '🤖 engineer-bot looked into this but made **no change** — the behaviour appears already correct.\n\n'; printf '> %s\n' "$REASON"; } > "$BODY" ;;
            *)
              { printf '🤖 engineer-bot could **not** complete an automated fix. See the [workflow run](%s) for details.\n' "$RUN_URL"; } > "$BODY" ;;
          esac
          gh issue comment "$ISSUE" --repo "$REPO" --body-file "$BODY"