databricks · mani-mathur-arch · Jul 14, 2026 · Jul 15, 2026
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -243,10 +243,9 @@ jobs:
       # missed, then runs `CGO_ENABLED=1 go test -tags databricks_kernel ./...`.
       #
       # No warehouse creds here — this job runs only the tagged UNIT tests, so the
-      # live e2e / Thrift-parity tests self-skip (they need DATABRICKS_HOST /
-      # DATABRICKS_HTTP_PATH / DATABRICKS_TOKEN). This matches how build-and-test
-      # treats the Thrift path (pure-Go unit tests only); live integration tests for
-      # both backends run through the separate trigger-integration-tests.yml dispatch
-      # to databricks-driver-test, not in this workflow.
+      # live e2e / Thrift-parity tests self-skip (they need the DATABRICKS_PECOTESTING_*
+      # warehouse credentials). This matches how build-and-test treats the Thrift path
+      # (pure-Go unit tests only); the credentialed e2e suites for both backends run in
+      # the separate nightly-e2e workflow.
       - name: Build kernel lib + run tagged tests
         run: make test-kernel
diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
@@ -0,0 +1,244 @@
+name: Nightly E2E
+
+# Runs the credential-gated end-to-end suites against a real test SQL warehouse.
+# These tests self-skip in the ordinary "Go" workflow (which injects no warehouse
+# secrets), so real-warehouse behaviour — large multi-page CloudFetch, S3 downloads,
+# drain-past-deadline, real auth — is otherwise not exercised in CI.
+#
+# Covers both backends against one test warehouse:
+#   - Thrift (default pure-Go) E2E — driver_e2e_test.go
+#   - SEA-via-kernel E2E          — kernel_e2e_test.go (cgo + databricks_kernel)
+#
+# Both suites read the same DATABRICKS_PECOTESTING_* warehouse credentials, so a
+# single secret set drives the whole workflow. A failed scheduled run surfaces via
+# GitHub's built-in notification.
+
+on:
+  schedule:
+    # 07:00 UTC daily (~midnight PT / mid-morning IST) — off-peak for the shared
+    # test warehouse. Scheduled workflows only run on the default branch.
+    - cron: '0 7 * * *'
+  # Manual trigger for on-demand runs (e.g. validating a fix or new secret wiring)
+  # without waiting for the nightly cron.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  thrift-e2e:
+    name: E2E (Thrift backend)
+    # Skips cleanly (no-op success) if the warehouse secret isn't provisioned yet,
+    # so the workflow doesn't hard-fail on a repo where the secrets haven't landed.
+    if: ${{ vars.NIGHTLY_E2E_ENABLED != 'false' }}
+    timeout-minutes: 20
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Setup JFrog
+        uses: ./.github/actions/setup-jfrog
+
+      - name: Set up Go Toolchain
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: '1.25.x'
+          cache: false
+
+      - name: Get dependencies
+        run: |
+          if ! command -v make &> /dev/null ; then
+            apt-get update && apt-get install -y make
+          fi
+          go get -v -t -d ./...
+
+      # Guard: fail loudly if a credential is missing rather than letting the tests
+      # silently t.Skip() into a misleading green. The token may be provided as either
+      # _TOKEN or _TOKEN_PERSONAL (the tests accept either), so require at least one.
+      - name: Verify warehouse credentials are present
+        env:
+          HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then
+            echo "::error::Nightly E2E warehouse secrets are not set (need DATABRICKS_PECOTESTING_SERVER_HOSTNAME, _HTTP_PATH2, and _TOKEN or _TOKEN_PERSONAL). Provision them in the repository settings." >&2
+            exit 1
+          fi
+
+      # The Thrift E2E tests read the DATABRICKS_PECOTESTING_* vars directly and run
+      # in the default pure-Go build (CGO_ENABLED=0). -run targets the high-value
+      # real-warehouse scenarios: large multi-page CloudFetch drain-past-caller-
+      # deadline and exact row count. -count=1 defeats the test cache so a scheduled
+      # run always re-exercises the warehouse. NOT -short, so
+      # TestE2ECloudFetchExactRowCount (guarded by -short) actually runs.
+      - name: Run Thrift E2E
+        env:
+          CGO_ENABLED: 0
+          DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          go test -count=1 -v -timeout 15m \
+            -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount' .
+
+  kernel-e2e:
+    name: E2E (kernel backend)
+    if: ${{ vars.NIGHTLY_E2E_ENABLED != 'false' }}
+    # Bounds the same source-build blast radius as the go.yml kernel job: an external
+    # clone + ~200-crate cold compile shouldn't hold a protected-runner slot to the
+    # 360-min default. Budget is split explicitly (see the per-step timeout on "Build
+    # kernel lib" below) so the job cap sits strictly above setup + build + the go
+    # test -timeout, letting Go's own timeout fire first and emit its which-test-hung
+    # goroutine dump instead of GitHub SIGKILLing the job mid-build on a cache miss.
+    timeout-minutes: 65
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      # Go module proxy (GOPROXY + ~/.netrc) via JFrog OIDC. Also exports
+      # JFROG_ACCESS_TOKEN, which the cargo step below reuses.
+      - name: Setup JFrog
+        uses: ./.github/actions/setup-jfrog
+
+      # The protected runner blocks direct crates.io access (go/hardened-gha), so
+      # point cargo at the JFrog crates proxy, reusing setup-jfrog's JFROG_ACCESS_TOKEN.
+      - name: Configure cargo to use JFrog
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.cargo
+          cat > ~/.cargo/config.toml << 'EOF'
+          [source.crates-io]
+          replace-with = "jfrog"
+
+          [source.jfrog]
+          registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
+
+          [registries.jfrog]
+          index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/"
+          credential-provider = ["cargo:token"]
+          EOF
+          cat > ~/.cargo/credentials.toml << EOF
+          [registries.jfrog]
+          token = "Bearer ${JFROG_ACCESS_TOKEN}"
+          EOF
+          chmod 600 ~/.cargo/credentials.toml
+
+      - name: Set up Go Toolchain
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: '1.25.x'
+          cache: false
+
+      # Keep in lockstep with rust-toolchain.toml's channel (it governs the archive
+      # under a fixed KERNEL_REV; a floating `stable` here would drift it).
+      - name: Set up Rust Toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1
+        with:
+          toolchain: 1.96.1
+
+      # The kernel repo is private and the hardened runner has no ambient git creds.
+      # Mint a repo-scoped token from the INTEGRATION_TEST_APP GitHub App and rewrite
+      # the kernel HTTPS URL to carry it (same mechanism as the go.yml kernel job).
+      - name: Generate GitHub App token for databricks-sql-kernel
+        id: kernel-token
+        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        with:
+          app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }}
+          private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }}
+          owner: databricks
+          repositories: databricks-sql-kernel
+
+      - name: Rewrite kernel repo URL to authenticated HTTPS
+        env:
+          TOKEN: ${{ steps.kernel-token.outputs.token }}
+        run: |
+          git config --global \
+            url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \
+            "https://github.com/databricks/"
+
+      - name: Cache kernel static lib
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            internal/backend/kernel/lib
+            internal/backend/kernel/include
+          key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }}
+
+      - name: Cache cargo + kernel build tree
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            build/kernel-src/target
+          key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-kernel-cargo-
+
+      - name: Install build prerequisites (make, C compiler)
+        run: |
+          if ! command -v make &> /dev/null ; then
+            apt-get update && apt-get install -y make
+          fi
+          if ! command -v cc &> /dev/null ; then
+            apt-get update && apt-get install -y build-essential
+          fi
+
+      # Same guard as the Thrift job: require host, http path, and at least one of
+      # _TOKEN / _TOKEN_PERSONAL so a missing credential fails loud instead of skipping.
+      - name: Verify warehouse credentials are present
+        env:
+          HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then
+            echo "::error::Nightly E2E warehouse secrets are not set (need DATABRICKS_PECOTESTING_SERVER_HOSTNAME, _HTTP_PATH2, and _TOKEN or _TOKEN_PERSONAL). Provision them in the repository settings." >&2
+            exit 1
+          fi
+
+      # Build the pinned kernel .a (make kernel-lib, cache permitting). Cap the Rust
+      # build explicitly so a cold ~200-crate compile that wedges is killed here as a
+      # build failure, rather than eating into the go test budget and letting GitHub
+      # SIGKILL the whole job before Go's -timeout can report which test hung. 25m
+      # (build) + 30m (go test) + setup stays under the 65m job cap.
+      - name: Build kernel lib
+        timeout-minutes: 25
+        run: make kernel-lib
+
+      # Run the kernel E2E tests tagged cgo+databricks_kernel. The kernel suite reads
+      # the same DATABRICKS_PECOTESTING_* credentials as the Thrift suite. -run selects
+      # the kernel E2E funcs plus the Thrift-vs-kernel parity funcs (TestKernelThriftParity
+      # / TestKernelParamsVsThrift) — the latter need both the kernel lib AND live
+      # credentials, so this is the only job that can run them. The tagged UNIT tests
+      # already run credential-free in the go.yml kernel job.
+      #
+      # TestKernelE2EM2M is excluded: -run matches it (unanchored prefix), but it needs
+      # a service principal (DATABRICKS_CLIENT_ID / _CLIENT_SECRET) that this warehouse
+      # secret set doesn't carry, so it would t.Skip() every run. -skip keeps it clearly
+      # out of scope here rather than a misleading silent skip; add M2M service-principal
+      # secrets + drop the -skip to cover it.
+      - name: Run kernel E2E
+        env:
+          DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }}
+          DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }}
+          DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }}
+          DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
+        run: |
+          CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \
+            -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \
+            -skip 'TestKernelE2EM2M' .
diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go
@@ -18,10 +18,10 @@ import (
 	dbsqlerr "github.com/databricks/databricks-sql-go/errors"
 )
 
-// kernelTestDB opens a kernel-backed *sql.DB from DATABRICKS_HOST /
-// DATABRICKS_HTTP_PATH / DATABRICKS_TOKEN, or skips when they are unset. It goes
-// through the standard connector with WithUseKernel(true) — the same path a real
-// consumer uses — not a kernel-only connector.
+// kernelTestDB opens a kernel-backed *sql.DB from the DATABRICKS_PECOTESTING_*
+// warehouse credentials, or skips when they are unset. It goes through the standard
+// connector with WithUseKernel(true) — the same path a real consumer uses — not a
+// kernel-only connector.
 func kernelTestDB(t *testing.T) *sql.DB {
 	t.Helper()
 	return kernelTestDBWith(t)
@@ -47,11 +47,16 @@ func TestKernelE2ESelect1(t *testing.T) {
 // counterpart to kernelTestDB.
 func kernelTestDBWith(t *testing.T, extra ...ConnOption) *sql.DB {
 	t.Helper()
-	host := os.Getenv("DATABRICKS_HOST")
-	httpPath := os.Getenv("DATABRICKS_HTTP_PATH")
-	token := os.Getenv("DATABRICKS_TOKEN")
+	// Reads the same DATABRICKS_PECOTESTING_* warehouse credentials as the Thrift
+	// E2E suite, so both backends run against one test warehouse from one secret set.
+	host := os.Getenv("DATABRICKS_PECOTESTING_SERVER_HOSTNAME")
+	httpPath := os.Getenv("DATABRICKS_PECOTESTING_HTTP_PATH2")
+	token := os.Getenv("DATABRICKS_PECOTESTING_TOKEN")
+	if token == "" {
+		token = os.Getenv("DATABRICKS_PECOTESTING_TOKEN_PERSONAL")
+	}
 	if host == "" || httpPath == "" || token == "" {
-		t.Skip("set DATABRICKS_HOST / DATABRICKS_HTTP_PATH / DATABRICKS_TOKEN for the kernel e2e")
+		t.Skip("set DATABRICKS_PECOTESTING_SERVER_HOSTNAME, DATABRICKS_PECOTESTING_HTTP_PATH2, and DATABRICKS_PECOTESTING_TOKEN for the kernel e2e")
 	}
 	opts := append([]ConnOption{
 		WithServerHostname(host),
@@ -471,12 +476,12 @@ func TestKernelE2EMetricViewMetadata(t *testing.T) {
 // M2M setter path actually authenticates end to end (not just that set_auth_m2m
 // returns OK, which TestSetAuthByMode already covers).
 func TestKernelE2EM2M(t *testing.T) {
-	host := os.Getenv("DATABRICKS_HOST")
-	httpPath := os.Getenv("DATABRICKS_HTTP_PATH")
+	host := os.Getenv("DATABRICKS_PECOTESTING_SERVER_HOSTNAME")
+	httpPath := os.Getenv("DATABRICKS_PECOTESTING_HTTP_PATH2")
 	clientID := os.Getenv("DATABRICKS_CLIENT_ID")
 	clientSecret := os.Getenv("DATABRICKS_CLIENT_SECRET")
 	if host == "" || httpPath == "" || clientID == "" || clientSecret == "" {
-		t.Skip("set DATABRICKS_HOST / DATABRICKS_HTTP_PATH / DATABRICKS_CLIENT_ID / DATABRICKS_CLIENT_SECRET for the M2M e2e")
+		t.Skip("set DATABRICKS_PECOTESTING_SERVER_HOSTNAME, DATABRICKS_PECOTESTING_HTTP_PATH2, DATABRICKS_CLIENT_ID, and DATABRICKS_CLIENT_SECRET for the M2M e2e")
 	}
 
 	connector, err := NewConnector(

diff --git a/kernel_parity_test.go b/kernel_parity_test.go
@@ -13,11 +13,14 @@ import (
 // parity comparison against the kernel backend.
 func thriftTestDB(t *testing.T) *sql.DB {
 	t.Helper()
-	host := os.Getenv("DATABRICKS_HOST")
-	httpPath := os.Getenv("DATABRICKS_HTTP_PATH")
-	token := os.Getenv("DATABRICKS_TOKEN")
+	host := os.Getenv("DATABRICKS_PECOTESTING_SERVER_HOSTNAME")
+	httpPath := os.Getenv("DATABRICKS_PECOTESTING_HTTP_PATH2")
+	token := os.Getenv("DATABRICKS_PECOTESTING_TOKEN")
+	if token == "" {
+		token = os.Getenv("DATABRICKS_PECOTESTING_TOKEN_PERSONAL")
+	}
 	if host == "" || httpPath == "" || token == "" {
-		t.Skip("set DATABRICKS_HOST / DATABRICKS_HTTP_PATH / DATABRICKS_TOKEN for the parity test")
+		t.Skip("set DATABRICKS_PECOTESTING_SERVER_HOSTNAME, DATABRICKS_PECOTESTING_HTTP_PATH2, and DATABRICKS_PECOTESTING_TOKEN for the parity test")
 	}
 	connector, err := NewConnector(
 		WithServerHostname(host),