bench.yml

# Runs after every commit to `develop` (or in other words, _after_ every pull request merges).

name: Benchmarks

on:
  push:
    branches: [develop]

permissions:
  id-token: write  # enables AWS-GitHub OIDC
  actions: read
  contents: write

jobs:
  commit-metadata:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
      - name: Setup AWS CLI
        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b  # v6
        with:
          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
          aws-region: us-east-1
      - name: Upload Commit Metadata
        shell: bash
        run: |
          set -Eeu -o pipefail -x

          sudo apt-get update && sudo apt-get install -y jq
          bash scripts/commit-json.sh > new-commit.json
          bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json

  bench:
    timeout-minutes: 120
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
          || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        benchmark:
          - id: random-access-bench
            name: Random Access
            build_args: "--features lance"
            formats: "parquet,lance,vortex"
          - id: compress-bench
            name: Compression
            build_args: "--features lance"
            formats: "parquet,lance,vortex"
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
      - name: Setup benchmark environment
        run: sudo bash scripts/setup-benchmark.sh
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install DuckDB
        run: |
          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.3/duckdb_cli-linux-amd64.zip | funzip > duckdb
          chmod +x duckdb
          echo "$PWD" >> $GITHUB_PATH

      - uses: ./.github/actions/system-info

      - name: Build binary
        shell: bash
        env:
          RUSTFLAGS: "-C target-cpu=native"
        run: |
          cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings

      - name: Setup Polar Signals
        uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
        with:
          polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}"
          project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
          profiling_frequency: 199
          extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz

      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
        if: matrix.benchmark.id == 'random-access-bench'
        shell: bash
        env:
          RUST_BACKTRACE: full
          VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
        run: |
          python3 scripts/random-access-split.py --v3

      - name: Run ${{ matrix.benchmark.name }} benchmark
        if: matrix.benchmark.id != 'random-access-bench'
        shell: bash
        env:
          RUST_BACKTRACE: full
          VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
        run: |
          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json --gh-json-v3 results.v3.jsonl

      - name: Setup AWS CLI
        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b  # v6
        with:
          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
          aws-region: us-east-1

      - name: Upload Benchmark Results
        shell: bash
        run: |
          bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json

      - name: Ingest results to v3 server
        if: vars.V3_INGEST_URL != ''
        shell: bash
        env:
          INGEST_BEARER_TOKEN: ${{ secrets.INGEST_BEARER_TOKEN }}
        run: |
          python3 scripts/post-ingest.py results.v3.jsonl \
            --server "${{ vars.V3_INGEST_URL }}" \
            --commit-sha "${{ github.sha }}" \
            --benchmark-id "${{ matrix.benchmark.id }}" \
            --repo-url "${{ github.server_url }}/${{ github.repository }}"

      # v4 (Postgres) dual-write -- BEST-EFFORT during the migration soak. The
      # proven v3 step above is hard-required; a v4 failure must NOT fail the job
      # (v4 is promoted to required at cutover, PR-5.1). Gated on the ingest-role
      # ARN var (the assume-role input that MUST exist for OIDC to succeed) so it
      # no-ops until v4 infra is wired, and every step is
      # `continue-on-error` so an OIDC / uv / connect hiccup never breaks the v3
      # pipeline. post-ingest.py mints the RDS IAM token internally (boto3) from
      # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert.
      #
      # ORDER MATTERS: "Install uv" runs BEFORE "Configure AWS credentials".
      # configure-aws-credentials persists the assumed ingest-role (rds-db:connect
      # only) as the job's ambient AWS creds; the uv setup compiles via sccache
      # (S3-backed), so running it after the role switch fails with S3 AccessDenied.
      # Installing uv first keeps sccache on the original S3-capable creds; the role
      # is assumed immediately before the ingest, which needs only rds-db:connect.
      - name: Install uv for v4 ingest
        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
        continue-on-error: true
        uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
      - name: Configure AWS credentials for v4 ingest (OIDC)
        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
        continue-on-error: true
        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b  # v6
        with:
          role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }}
          aws-region: ${{ vars.RDS_BENCH_REGION }}
      - name: Ingest results to v4 Postgres (best-effort)
        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
        continue-on-error: true
        shell: bash
        env:
          RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
          RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }}
          AWS_REGION: ${{ vars.RDS_BENCH_REGION }}
          BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }}
          BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}
        run: |
          set -Eeuo pipefail
          curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
            -o "${RUNNER_TEMP}/rds-global-bundle.pem"
          DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
          uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \
            scripts/post-ingest.py results.v3.jsonl \
            --postgres "${DSN}" \
            --commit-sha "${{ github.sha }}" \
            --region "${AWS_REGION}"

      - name: Alert incident.io
        if: failure()
        uses: ./.github/actions/alert-incident-io
        with:
          api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
          alert-title: "${{ matrix.benchmark.name }} benchmark failed on develop"
          deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure

  sql:
    uses: ./.github/workflows/sql-benchmarks.yml
    secrets: inherit
    with:
      mode: "develop"