feat(kernel): consolidated DAIS gap-closure + PuPr feature set + nightly workflows for e2e tests

.github/workflows/go.yml

@@ -243,10 +243,9 @@ jobs:
       # missed, then runs `CGO_ENABLED=1 go test -tags databricks_kernel ./...`.
       #
       # No warehouse creds here — this job runs only the tagged UNIT tests, so the
-      # live e2e / Thrift-parity tests self-skip (they need DATABRICKS_HOST /
-      # DATABRICKS_HTTP_PATH / DATABRICKS_TOKEN). This matches how build-and-test
-      # treats the Thrift path (pure-Go unit tests only); live integration tests for
-      # both backends run through the separate trigger-integration-tests.yml dispatch
-      # to databricks-driver-test, not in this workflow.
+      # live e2e / Thrift-parity tests self-skip (they need the DATABRICKS_PECOTESTING_*
+      # warehouse credentials). This matches how build-and-test treats the Thrift path
+      # (pure-Go unit tests only); the credentialed e2e suites for both backends run in
+      # the separate nightly-e2e workflow.
       - name: Build kernel lib + run tagged tests
         run: make test-kernel

.github/workflows/nightly-e2e.yml

@@ -0,0 +1,246 @@
+name: Nightly E2E
+
+# Runs the credential-gated end-to-end suites against a real test SQL warehouse.
+# These tests self-skip in the ordinary "Go" workflow (which injects no warehouse
+# secrets), so real-warehouse behaviour — large multi-page CloudFetch, S3 downloads,
+# drain-past-deadline, real auth — is otherwise not exercised in CI.
+#
+# Covers both backends against one test warehouse:
+#   - Thrift (default pure-Go) E2E — driver_e2e_test.go
+#   - SEA-via-kernel E2E          — kernel_e2e_test.go (cgo + databricks_kernel)
+#
+# Both suites read the same DATABRICKS_PECOTESTING_* warehouse credentials, so a
+# single secret set drives the whole workflow. A failed scheduled run surfaces via
+# GitHub's built-in notification.
+
+on:
+  schedule:
+    # 07:00 UTC daily (~midnight PT / mid-morning IST) — off-peak for the shared
+    # test warehouse. Scheduled workflows only run on the default branch.
+    - cron: '0 7 * * *'
+  # Manual trigger for on-demand runs (e.g. validating a fix or new secret wiring)
+  # without waiting for the nightly cron.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  thrift-e2e:
+    name: E2E (Thrift backend)
+    # Runs by default; disable by setting the NIGHTLY_E2E_ENABLED repo variable to
+    # 'false'. This is NOT a skip-on-missing-secret gate: if the variable is unset
+    # and the warehouse secrets aren't provisioned, the credential step below
+    # hard-fails by design (a silent skip reporting green is the gap being closed).
+    if: ${{ vars.NIGHTLY_E2E_ENABLED != 'false' }}
+    timeout-minutes: 20
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Setup JFrog
+        uses: ./.github/actions/setup-jfrog
+
+      - name: Set up Go Toolchain
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: '1.25.x'
+          cache: false
+
+      - name: Get dependencies
+        run: |
+          if ! command -v make &> /dev/null ; then
+            apt-get update && apt-get install -y make
+          fi
+          go get -v -t -d ./...
+
+      # Guard: fail loudly if a credential is missing rather than letting the tests
+      # silently t.Skip() into a misleading green. The token may be provided as either
+      # _TOKEN or _TOKEN_PERSONAL (the tests accept either), so require at least one.
+      - name: Verify warehouse credentials are present
+        env:
+          HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then
+            echo "::error::Nightly E2E warehouse secrets are not set (need DATABRICKS_PECOTESTING_SERVER_HOSTNAME, _HTTP_PATH2, and _TOKEN or _TOKEN_PERSONAL). Provision them in the repository settings." >&2
+            exit 1
+          fi
+
+      # The Thrift E2E tests read the DATABRICKS_PECOTESTING_* vars directly and run
+      # in the default pure-Go build (CGO_ENABLED=0). -run targets the high-value
+      # real-warehouse scenarios: large multi-page CloudFetch drain-past-caller-
+      # deadline and exact row count. -count=1 defeats the test cache so a scheduled
+      # run always re-exercises the warehouse. NOT -short, so
+      # TestE2ECloudFetchExactRowCount (guarded by -short) actually runs.
+      - name: Run Thrift E2E
+        env:
+          CGO_ENABLED: 0
+          DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          go test -count=1 -v -timeout 15m \
+            -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount' .
+
+  kernel-e2e:
+    name: E2E (kernel backend)
+    if: ${{ vars.NIGHTLY_E2E_ENABLED != 'false' }}
+    # Bounds the same source-build blast radius as the go.yml kernel job: an external
+    # clone + ~200-crate cold compile shouldn't hold a protected-runner slot to the
+    # 360-min default. Budget is split explicitly (see the per-step timeout on "Build
+    # kernel lib" below) so the job cap sits strictly above setup + build + the go
+    # test -timeout, letting Go's own timeout fire first and emit its which-test-hung
+    # goroutine dump instead of GitHub SIGKILLing the job mid-build on a cache miss.
+    timeout-minutes: 65
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      # Go module proxy (GOPROXY + ~/.netrc) via JFrog OIDC. Also exports
+      # JFROG_ACCESS_TOKEN, which the cargo step below reuses.
+      - name: Setup JFrog
+        uses: ./.github/actions/setup-jfrog
+
+      # The protected runner blocks direct crates.io access (go/hardened-gha), so
+      # point cargo at the JFrog crates proxy, reusing setup-jfrog's JFROG_ACCESS_TOKEN.
+      - name: Configure cargo to use JFrog
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.cargo
+          cat > ~/.cargo/config.toml << 'EOF'
+          [source.crates-io]
+          replace-with = "jfrog"
+
+          [source.jfrog]
+          registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
+
+          [registries.jfrog]
+          index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
+          credential-provider = ["cargo:token"]
+          EOF
+          cat > ~/.cargo/credentials.toml << EOF
+          [registries.jfrog]
+          token = "Bearer ${JFROG_ACCESS_TOKEN}"
+          EOF
+          chmod 600 ~/.cargo/credentials.toml
+
+      - name: Set up Go Toolchain
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: '1.25.x'
+          cache: false
+
+      # Keep in lockstep with rust-toolchain.toml's channel (it governs the archive
+      # under a fixed KERNEL_REV; a floating `stable` here would drift it).
+      - name: Set up Rust Toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1
+        with:
+          toolchain: 1.96.1
+
+      # The kernel repo is private and the hardened runner has no ambient git creds.
+      # Mint a repo-scoped token from the INTEGRATION_TEST_APP GitHub App and rewrite
+      # the kernel HTTPS URL to carry it (same mechanism as the go.yml kernel job).
+      - name: Generate GitHub App token for databricks-sql-kernel
+        id: kernel-token
+        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        with:
+          app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }}
+          private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }}
+          owner: databricks
+          repositories: databricks-sql-kernel
+
+      - name: Rewrite kernel repo URL to authenticated HTTPS
+        env:
+          TOKEN: ${{ steps.kernel-token.outputs.token }}
+        run: |
+          git config --global \
+            url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \
+            "https://github.com/databricks/"
+
+      - name: Cache kernel static lib
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            internal/backend/kernel/lib
+            internal/backend/kernel/include
+          key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }}
+
+      - name: Cache cargo + kernel build tree
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            build/kernel-src/target
+          key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-kernel-cargo-
+
+      - name: Install build prerequisites (make, C compiler)
+        run: |
+          if ! command -v make &> /dev/null ; then
+            apt-get update && apt-get install -y make
+          fi
+          if ! command -v cc &> /dev/null ; then
+            apt-get update && apt-get install -y build-essential
+          fi
+
+      # Same guard as the Thrift job: require host, http path, and at least one of
+      # _TOKEN / _TOKEN_PERSONAL so a missing credential fails loud instead of skipping.
+      - name: Verify warehouse credentials are present
+        env:
+          HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then
+            echo "::error::Nightly E2E warehouse secrets are not set (need DATABRICKS_PECOTESTING_SERVER_HOSTNAME, _HTTP_PATH2, and _TOKEN or _TOKEN_PERSONAL). Provision them in the repository settings." >&2
+            exit 1
+          fi
+
+      # Build the pinned kernel .a (make kernel-lib, cache permitting). Cap the Rust
+      # build explicitly so a cold ~200-crate compile that wedges is killed here as a
+      # build failure, rather than eating into the go test budget and letting GitHub
+      # SIGKILL the whole job before Go's -timeout can report which test hung. 25m
+      # (build) + 30m (go test) + setup stays under the 65m job cap.
+      - name: Build kernel lib
+        timeout-minutes: 25
+        run: make kernel-lib
+
+      # Run the kernel E2E tests tagged cgo+databricks_kernel. The kernel suite reads
+      # the same DATABRICKS_PECOTESTING_* credentials as the Thrift suite. -run selects
+      # the kernel E2E funcs plus the Thrift-vs-kernel parity funcs (TestKernelThriftParity
+      # / TestKernelParamsVsThrift) — the latter need both the kernel lib AND live
+      # credentials, so this is the only job that can run them. The tagged UNIT tests
+      # already run credential-free in the go.yml kernel job.
+      #
+      # TestKernelE2EM2M is excluded: -run matches it (unanchored prefix), but it needs
+      # a service principal (DATABRICKS_CLIENT_ID / _CLIENT_SECRET) that this warehouse
+      # secret set doesn't carry, so it would t.Skip() every run. -skip keeps it clearly
+      # out of scope here rather than a misleading silent skip; add M2M service-principal
+      # secrets + drop the -skip to cover it.
+      - name: Run kernel E2E
+        env:
+          DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \
+            -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \
+            -skip 'TestKernelE2EM2M' .

.github/workflows/trigger-integration-tests.yml

@@ -9,9 +9,18 @@ name: Trigger Integration Tests
 # preview with the `integration-test` label (replay) or `integration-test-full`
 # (full passthrough), and merge queue runs the real required gate.
 #
+# The suite runs against either driver backend, selected by the label:
+#   - `integration-test` (replay) / `integration-test-full` (passthrough) → Thrift
+#   - `integration-test-kernel` (passthrough)                             → SEA-via-kernel
+# The kernel label adds `go_mode: sea` to the dispatch payload; driver-test then
+# builds the kernel static lib and runs the tagged (databricks_kernel) leg. It uses
+# passthrough (real warehouse) — the sea leg has no committed recordings to replay
+# yet, so there is no sea replay label until those are captured.
+#
 # Required external setup:
 #
-#   1. `integration-test` and `integration-test-full` labels exist in this repo.
+#   1. The three trigger labels exist in this repo: `integration-test`,
+#      `integration-test-full`, `integration-test-kernel`.
 #   2. `INTEGRATION_TEST_APP_ID` / `INTEGRATION_TEST_PRIVATE_KEY` repo secrets
 #      are installed in this repo for the dispatcher GitHub App.
 #   3. The app is installed/granted on `databricks-driver-test` so this workflow
@@ -46,7 +55,7 @@ jobs:
         with:
           script: |
             const present = context.payload.pull_request.labels.map((l) => l.name);
-            const triggerLabels = ['integration-test', 'integration-test-full'];
+            const triggerLabels = ['integration-test', 'integration-test-full', 'integration-test-kernel'];
             const removed = [];
             for (const name of triggerLabels) {
               if (!present.includes(name)) continue;
@@ -111,7 +120,7 @@ jobs:
               completed_at: new Date().toISOString(),
               output: {
                 title: 'Skipped on PR - runs in merge queue',
-                summary: 'Go integration tests are skipped on ordinary PR events and run as a required gate in the merge queue. Add the `integration-test` label to preview replay (or `integration-test-full` for the full passthrough suite) on this PR.',
+                summary: 'Go integration tests are skipped on ordinary PR events and run as a required gate in the merge queue. Preview on this PR by adding a label: `integration-test` (Thrift, replay) or `integration-test-full` (Thrift, passthrough); `integration-test-kernel` runs the SEA-via-kernel backend against the real warehouse (passthrough).',
               },
             });
 
@@ -122,7 +131,8 @@ jobs:
       github.event_name == 'pull_request' &&
       github.event.action == 'labeled' &&
       (github.event.label.name == 'integration-test' ||
-       github.event.label.name == 'integration-test-full')
+       github.event.label.name == 'integration-test-full' ||
+       github.event.label.name == 'integration-test-kernel')
     runs-on:
       group: databricks-protected-runner-group
       labels: linux-ubuntu-latest
@@ -134,14 +144,24 @@ jobs:
       pull-requests: write
       checks: write
     steps:
-      - name: Resolve proxy mode from label
+      - name: Resolve proxy mode + backend from label
         id: mode
+        # go_mode: integration-test-kernel runs the SEA-via-kernel backend; the others
+        # run Thrift. proxy_mode: integration-test-full and integration-test-kernel hit
+        # the real warehouse (passthrough); plain integration-test serves recordings
+        # (replay). The kernel label is passthrough because the sea leg has no committed
+        # recordings to replay yet. driver-test reads go_mode to decide whether to build
+        # the kernel lib and run the tagged leg.
         run: |
-          if [ "${{ github.event.label.name }}" = "integration-test-full" ]; then
-            echo "proxy_mode=passthrough" >> "$GITHUB_OUTPUT"
-          else
-            echo "proxy_mode=replay" >> "$GITHUB_OUTPUT"
-          fi
+          LABEL="${{ github.event.label.name }}"
+          case "$LABEL" in
+            integration-test-full|integration-test-kernel) echo "proxy_mode=passthrough" >> "$GITHUB_OUTPUT" ;;
+            *)                                             echo "proxy_mode=replay"      >> "$GITHUB_OUTPUT" ;;
+          esac
+          case "$LABEL" in
+            integration-test-kernel) echo "go_mode=sea"    >> "$GITHUB_OUTPUT" ;;
+            *)                       echo "go_mode=thrift" >> "$GITHUB_OUTPUT" ;;
+          esac
 
       - name: Generate GitHub App token (driver-test repo)
         id: app-token
@@ -156,6 +176,7 @@ jobs:
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           PROXY_MODE: ${{ steps.mode.outputs.proxy_mode }}
+          GO_MODE: ${{ steps.mode.outputs.go_mode }}
         with:
           github-token: ${{ steps.app-token.outputs.token }}
           script: |
@@ -170,9 +191,10 @@ jobs:
                 pr_repo: context.repo.owner + '/' + context.repo.repo,
                 pr_url: pr.html_url,
                 proxy_mode: process.env.PROXY_MODE,
+                go_mode: process.env.GO_MODE,
               },
             });
-            core.info(`Dispatched go-pr-test (${process.env.PROXY_MODE}) for PR #${pr.number} @ ${pr.head.sha}`);
+            core.info(`Dispatched go-pr-test (${process.env.GO_MODE}/${process.env.PROXY_MODE}) for PR #${pr.number} @ ${pr.head.sha}`);
 
       - name: Fail check on dispatch error
         if: failure()
@@ -198,13 +220,14 @@ jobs:
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           PROXY_MODE: ${{ steps.mode.outputs.proxy_mode }}
+          GO_MODE: ${{ steps.mode.outputs.go_mode }}
         with:
           script: |
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.issue.number,
-              body: `Go integration tests triggered (\`${process.env.PROXY_MODE}\`). [View workflow runs](https://github.com/databricks/databricks-driver-test/actions).`,
+              body: `Go integration tests triggered (\`${process.env.GO_MODE}\` / \`${process.env.PROXY_MODE}\`). [View workflow runs](https://github.com/databricks/databricks-driver-test/actions).`,
             });
 
   # Merge queue: the required gate. Runs replay once before merge.
@@ -261,9 +284,14 @@ jobs:
                 pr_repo: context.repo.owner + '/' + context.repo.repo,
                 pr_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}`,
                 proxy_mode: 'replay',
+                // The required merge-queue gate runs the Thrift backend only. The
+                // kernel (sea) leg is previewed on demand via the kernel labels but
+                // is not a required gate — it becomes one once the SEA backend ships
+                // in a released driver and its recordings are captured.
+                go_mode: 'thrift',
               },
             });
-            core.info(`Merge-queue dispatch go-pr-test (replay) for PR #${prNumber} @ ${process.env.HEAD_SHA}`);
+            core.info(`Merge-queue dispatch go-pr-test (thrift/replay) for PR #${prNumber} @ ${process.env.HEAD_SHA}`);
 
       - name: Fail check on dispatch error
         if: failure()