diff --git a/.github/workflows/appliance.yml b/.github/workflows/appliance.yml index c123d3c3..05762d77 100644 --- a/.github/workflows/appliance.yml +++ b/.github/workflows/appliance.yml @@ -1,8 +1,8 @@ name: Appliance # Builds the downloadable virtual appliances for ALL hypervisors in one job: customizes the Ubuntu 24.04 -# cloud image with libguestfs (no Hyper-V host / nested virt needed) into a Gen2/UEFI image with SQL Server -# Express + Dispatch baked in, emits it as VHDX (Hyper-V), OVA (VMware), and qcow2 (KVM/Proxmox), then boots +# cloud image with libguestfs (no Hyper-V host / nested virt needed) into a Gen2/UEFI image with PostgreSQL +# + Dispatch baked in, emits it as VHDX (Hyper-V), OVA (VMware), and qcow2 (KVM/Proxmox), then boots # it under QEMU/UEFI to prove the service comes up on first boot. Artifacts: dispatch-appliance-hyperv, # dispatch-appliance-vmware, dispatch-appliance-kvm. on: @@ -20,7 +20,7 @@ jobs: # single image, then uploads them as three separate downloads near the end. The name reflects that # it is not Hyper-V-only. build-appliances: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 @@ -135,7 +135,7 @@ jobs: sudo convert /tmp/screen.ppm "$1" 2>/dev/null && echo "captured $1" || echo "no screenshot" } ok=0 - for i in $(seq 1 90); do # up to ~8 min (KVM boot is fast; this is mostly SQL first-run) + for i in $(seq 1 90); do # up to ~8 min (KVM boot is fast; this is mostly PostgreSQL first-run) if curl -fsSk -m 3 https://localhost:8420/health >/dev/null 2>&1; then ok=1; break; fi [ "$i" = 18 ] && screenshot boot-90s.png # ~90s in: capture whatever's on screen sleep 5 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4b22d45..e8e27b66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,25 +10,29 @@ env: jobs: build-test: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] services: - # SQL Server 2022 service container (GitHub-hosted runners are free for this public repo). Same wire - # protocol / T-SQL as the dev Azure SQL Edge container; app code is unchanged. The Data test fixture - # waits up to 90s for the engine to accept connections, so no container health-cmd is needed. - sql: - image: mcr.microsoft.com/mssql/server:2022-latest + # PostgreSQL service container. The Data test fixture waits up to 90s for the engine to accept + # connections; the health-cmd gates job steps until pg_isready succeeds. + db: + image: postgres:17 env: - ACCEPT_EULA: "Y" - # Dev-only SA password. Override by setting the MSSQL_SA_PASSWORD repo/org secret; the literal is + # Dev-only password. Override by setting the POSTGRES_PASSWORD repo/org secret; the literal is # only a fallback and grants nothing beyond the ephemeral CI container. - MSSQL_SA_PASSWORD: ${{ secrets.MSSQL_SA_PASSWORD || 'Dispatch_Dev_Pass123' }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD || 'Dispatch_Dev_Pass123' }} ports: - - 1433:1433 + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 12 env: - # Drives the Dispatch.Data integration tests against the SQL service container. - DISPATCH_TEST_SQL: "Server=localhost,1433;User Id=sa;Password=${{ secrets.MSSQL_SA_PASSWORD || 'Dispatch_Dev_Pass123' }};TrustServerCertificate=True;Encrypt=True" + # Drives the Dispatch.Data integration tests against the Postgres service container. No Database= here; + # the fixture creates a uniquely-named database per run and drops it afterwards. + DISPATCH_TEST_SQL: "Host=localhost;Port=5432;Username=postgres;Password=${{ secrets.POSTGRES_PASSWORD || 'Dispatch_Dev_Pass123' }}" # Make the integration tests fail loudly if the connection string is ever missing, instead of skipping. DISPATCH_REQUIRE_SQL: "1" DOTNET_NOLOGO: "true" @@ -83,7 +87,7 @@ jobs: # Keep em dashes out of the tree: they break Windows PowerShell parsing (a mis-decoded em-dash byte # becomes a smart-quote string delimiter) and the project standardizes on ASCII '-'. no-em-dash: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 - name: Reject em dashes (use ASCII '-') @@ -103,7 +107,7 @@ jobs: trigger-appliance: needs: [build-test, no-em-dash] if: ${{ github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'feat/relay-engine-web-providers') }} - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] permissions: actions: write steps: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a64dd3cd..89798953 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,6 @@ name: Docker -# Builds the container image and runs the full stack (Dispatch + SQL via docker-compose), failing unless +# Builds the container image and runs the full stack (Dispatch + PostgreSQL via docker-compose), failing unless # the service comes up and serves /health - the container equivalent of the installer smoke tests. Docker is # native on the runner, so this needs no QEMU/nested virtualization. The release workflow builds+pushes the # multi-arch image; this proves it actually runs before that. @@ -20,24 +20,24 @@ on: jobs: build-and-smoke: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 - - name: Build image + start the stack (Dispatch + SQL) + - name: Build image + start the stack (Dispatch + PostgreSQL) run: docker compose up --build -d - name: Wait for the dashboard /health run: | ok=0 - for _ in $(seq 1 60); do # up to 5 min: image already built; SQL start + DB init + service + for _ in $(seq 1 60); do # up to 5 min: image already built; PostgreSQL start + DB init + service if curl -fsSk -m 5 https://localhost:8420/health >/dev/null 2>&1; then ok=1; break; fi sleep 5 done if [ "$ok" != 1 ]; then echo "=== compose ps ==="; docker compose ps echo "=== dispatch logs ==="; docker compose logs --tail 150 dispatch || true - echo "=== sql logs ==="; docker compose logs --tail 40 sql || true + echo "=== postgresql logs ==="; docker compose logs --tail 40 db || true echo "dispatch /health did not respond"; exit 1 fi echo "Docker stack /health OK" diff --git a/.github/workflows/installers.yml b/.github/workflows/installers.yml index 3ccd662e..0420a7a5 100644 --- a/.github/workflows/installers.yml +++ b/.github/workflows/installers.yml @@ -2,13 +2,13 @@ name: Installers # Validates the installer artifacts on GitHub-hosted runners (free for this public repo). Non-destructive # by default: the Windows job builds the MSI + Burn bundle (proving the WiX sources + net472 launcher -# compile) and the Linux job lints install.sh. The actual SQL-Express install + service smoke test is -# destructive (it installs SQL Server on the runner), so it only runs when dispatched with full_install=true. +# compile) and the Linux job lints install.sh. The actual PostgreSQL install + service smoke test is +# destructive (it installs PostgreSQL on the runner), so it only runs when dispatched with full_install=true. on: workflow_dispatch: inputs: full_install: - description: "Also run the real SQL install + service smoke test (mutates the runner)" + description: "Also run the real PostgreSQL install + service smoke test (mutates the runner)" type: boolean default: false push: @@ -24,7 +24,7 @@ env: jobs: windows-installer: - runs-on: windows-latest + runs-on: [self-hosted, windows, x64] steps: - uses: actions/checkout@v5 @@ -83,19 +83,19 @@ jobs: working-directory: installer/windows run: wix build Dispatch.wxs -arch x64 -d PublishDir=publish -ext WixToolset.Firewall.wixext -o Dispatch.msi - - name: Build the SQL Express launcher - run: dotnet build installer/windows/sql-express/SqlExpressLauncher.csproj -c Release + - name: Build the PostgreSQL launcher + run: dotnet build installer/windows/postgres/PostgresLauncher.csproj -c Release - name: Build the bundle working-directory: installer/windows shell: pwsh run: | wix build bundle/Bundle.wxs -ext WixToolset.BootstrapperApplications.wixext -ext WixToolset.Util.wixext ` - -bindpath "SqlLauncher=sql-express/bin/Release/net472" -bindpath "Msi=." -o DispatchSetup.exe + -bindpath "PgLauncher=postgres/bin/Release/net472" -bindpath "Msi=." -o DispatchSetup.exe # CI installer builds are intentionally unsigned (signing happens only on tagged releases via # release.yml + Azure Artifact Signing). This artifact proves the WiX sources + launcher compile. - # DispatchSetup.exe is the single distributable file - it embeds Dispatch.msi and chains SQL Express. + # DispatchSetup.exe is the single distributable file - it embeds Dispatch.msi and chains PostgreSQL. # (The MSI is still built above to prove the WiX sources compile and to embed in the bundle.) - name: Upload installer artifact uses: actions/upload-artifact@v6 @@ -104,7 +104,7 @@ jobs: path: installer/windows/DispatchSetup.exe if-no-files-found: error - # Real end-to-end check: run the bundle (installs SQL Express + the service) and confirm the Windows + # Real end-to-end check: run the bundle (installs PostgreSQL + the service) and confirm the Windows # service starts and serves /health. Runs on every installer-path push and on manual dispatch, so a # service that won't start (bad config, missing files, crash on boot) fails CI instead of the customer. - name: Full install + smoke @@ -246,7 +246,7 @@ jobs: # the Actions run page). This needs NO tag and creates NO release - releases are only for permanent # public download links. Grab "dispatch-linux-tarball" from the run to test install.sh --prebuilt. linux-tarball: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 @@ -291,9 +291,12 @@ jobs: stage="dispatch-ci-linux-x64" mkdir -p "staging/${stage}/bin" cp -r publish/linux/. "staging/${stage}/bin/" - cp installer/linux/install.sh "staging/${stage}/install.sh" - cp installer/linux/dispatch.service "staging/${stage}/dispatch.service" - chmod +x "staging/${stage}/install.sh" "staging/${stage}/bin/Dispatch.Service" + # Ship the same files as the release tarball (install.sh installs all of these systemd units). + cp installer/linux/install.sh installer/linux/dispatch.service \ + installer/linux/dispatch-updater.service installer/linux/dispatch-update.path \ + installer/linux/dispatch-update.sh "staging/${stage}/" + cp src/Dispatch.Core/Updates/dispatch-update-public.pem "staging/${stage}/" + chmod +x "staging/${stage}/install.sh" "staging/${stage}/dispatch-update.sh" "staging/${stage}/bin/Dispatch.Service" tar -czf "${stage}.tar.gz" -C staging "${stage}" ls -lh "${stage}.tar.gz" @@ -305,20 +308,20 @@ jobs: if-no-files-found: error # Real end-to-end check (mirrors the Windows full-install smoke): install the just-built self-contained - # tarball exactly as a user would - install.sh --prebuilt --install-sql, which installs SQL Server + # tarball exactly as a user would - install.sh --prebuilt --install-postgres, which installs PostgreSQL # locally and provisions the DispatchLog DB - then confirm the systemd service starts and serves # /health. Runs on installer-path pushes and manual dispatch so a service that won't start (or a broken - # SQL bootstrap) fails CI instead of the customer. + # PostgreSQL bootstrap) fails CI instead of the customer. - name: Full install + smoke if: ${{ github.event_name == 'push' || inputs.full_install == true }} env: - SA_PW: "Ci-Smoke-Passw0rd!" + DB_PW: "Ci-Smoke-Passw0rd!" run: | set -euo pipefail mkdir -p /tmp/extract && tar -xzf dispatch-ci-linux-x64.tar.gz -C /tmp/extract dir=/tmp/extract/dispatch-ci-linux-x64 - sudo "$dir/install.sh" --prebuilt "$dir/bin" --install-sql \ - --sa-password "$SA_PW" --admin-password "$SA_PW" + sudo "$dir/install.sh" --prebuilt "$dir/bin" --install-postgres \ + --db-password "$DB_PW" --admin-password "$DB_PW" ok=0 for _ in $(seq 1 36); do @@ -329,7 +332,7 @@ jobs: echo "=== dispatch service ==="; sudo systemctl status dispatch --no-pager || true echo "=== journal ==="; sudo journalctl -u dispatch --no-pager -n 80 || true echo "=== app logs ==="; sudo tail -n 60 /var/log/dispatch/*.log 2>/dev/null || true - echo "=== sql server ==="; sudo systemctl status mssql-server --no-pager 2>/dev/null || true + echo "=== postgresql ==="; sudo systemctl status postgresql --no-pager 2>/dev/null || true echo "health check failed"; exit 1 fi curl -fsSk https://localhost:8420/health; echo " <- /health OK" @@ -347,7 +350,7 @@ jobs: [ "$tls_ok" = 1 ] || { echo "STARTTLS negotiation failed on all SMTP ports"; exit 1; } linux-installer: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 @@ -361,5 +364,6 @@ jobs: fi # The end-to-end install + "service actually runs" smoke lives in the linux-tarball job below - it - # installs the real self-contained tarball (install.sh --prebuilt) against a SQL container and checks - # /health, mirroring the Windows full-install smoke. This job stays a fast lint of install.sh. + # installs the real self-contained tarball (install.sh --prebuilt) against a locally-installed + # PostgreSQL and checks /health, mirroring the Windows full-install smoke. This job stays a fast lint + # of install.sh. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be404a94..c5bfba58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ name: Release # # This builds the distributable artifacts, stamps them with the tag version, optionally Authenticode-signs # the Windows installer via Azure Artifact Signing, and publishes a GitHub Release with: -# - DispatchSetup.exe the single-file Windows installer (embeds the signed MSI + chains SQL Express) +# - DispatchSetup.exe the single-file Windows installer (embeds the signed MSI + chains PostgreSQL) # - dispatch--linux.tar.gz universal Linux bundle (x64 + arm64, self-contained) # - SHA256SUMS checksums for all of the above # @@ -40,7 +40,7 @@ env: jobs: windows: - runs-on: windows-latest + runs-on: [self-hosted, windows, x64] outputs: version: ${{ steps.ver.outputs.version }} steps: @@ -116,8 +116,8 @@ jobs: timestamp-rfc3161: http://timestamp.acs.microsoft.com timestamp-digest: SHA256 - - name: Build the SQL Express launcher - run: dotnet build installer/windows/sql-express/SqlExpressLauncher.csproj -c Release + - name: Build the PostgreSQL launcher + run: dotnet build installer/windows/postgres/PostgresLauncher.csproj -c Release - name: Build the bundle working-directory: installer/windows @@ -125,12 +125,12 @@ jobs: run: | wix build bundle/Bundle.wxs -d Version=${{ steps.ver.outputs.version }} ` -ext WixToolset.BootstrapperApplications.wixext -ext WixToolset.Util.wixext ` - -bindpath "SqlLauncher=sql-express/bin/Release/net472" -bindpath "Msi=." -o DispatchSetup.exe + -bindpath "PgLauncher=postgres/bin/Release/net472" -bindpath "Msi=." -o DispatchSetup.exe # --- Sign the Burn bundle (opt-in) -------------------------------------------------------- # A Burn bundle has an attached engine; the engine must be signed too, so detach it, sign it, # reattach, then sign the final bundle (WiX-recommended; otherwise UAC flags the inner engine). - # installer/windows holds other .exe files (publish/, sql-express/), so each Azure sign step + # installer/windows holds other .exe files (publish/, postgres/), so each Azure sign step # targets a folder containing ONLY the one file to sign, and we copy the result back in place. - name: Detach the bundle engine if: ${{ vars.AZURE_SIGNING_ACCOUNT != '' }} @@ -185,12 +185,12 @@ jobs: uses: actions/upload-artifact@v6 with: name: windows - # DispatchSetup.exe is the single distributable - it embeds the (signed) MSI and chains SQL Express. + # DispatchSetup.exe is the single distributable - it embeds the (signed) MSI and chains PostgreSQL. path: installer/windows/DispatchSetup.exe if-no-files-found: error linux: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 @@ -247,11 +247,10 @@ jobs: Dispatch SMTP Relay ${ver} - Linux (universal: x64 + arm64, self-contained; no .NET SDK required). install.sh auto-detects your CPU arch (uses bin-x64 or bin-arm64) - no flags needed: - sudo ./install.sh --install-sql --sa-password '' --admin-password '' + sudo ./install.sh --install-postgres --db-password '' --admin-password '' - On arm64, --install-sql runs Azure SQL Edge in a container (SQL Server is amd64-only). - Or use an existing SQL Server / Azure SQL: - sudo ./install.sh --sql-connection 'Server=...;Database=DispatchLog;User Id=...;Password=...;TrustServerCertificate=True;Encrypt=True' --admin-password '' + Or use an existing PostgreSQL server: + sudo ./install.sh --sql-connection 'Host=...;Port=5432;Database=DispatchLog;Username=...;Password=...' --admin-password '' Then open https://localhost:8420 (self-signed cert) and log in with the admin password you set. EOF @@ -311,7 +310,7 @@ jobs: # Only runs for real tag releases (a dry-run workflow_dispatch skips the push). docker: if: ${{ github.ref_type == 'tag' }} - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 @@ -352,7 +351,7 @@ jobs: publish: needs: [windows, linux] - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - name: Download artifacts uses: actions/download-artifact@v8 diff --git a/.github/workflows/scripts.yml b/.github/workflows/scripts.yml index 46b93a51..5012bbdd 100644 --- a/.github/workflows/scripts.yml +++ b/.github/workflows/scripts.yml @@ -15,7 +15,7 @@ on: jobs: lint: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v5 - name: Parse + analyze PowerShell @@ -45,7 +45,7 @@ jobs: Write-Host 'OK: PowerShell parses and lints clean.' pester: - runs-on: windows-latest + runs-on: [self-hosted, windows, x64] steps: - uses: actions/checkout@v5 - name: Pester (Hyper-V import logic, cmdlets mocked) diff --git a/Dockerfile b/Dockerfile index 37867b71..ec9f604b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,12 @@ # Dispatch SMTP Relay - container image (multi-arch: linux/amd64 + linux/arm64). # # Build: docker build -t dispatch . -# Run: see docker-compose.yml (brings up Dispatch + SQL together). +# Run: see docker-compose.yml (brings up Dispatch + PostgreSQL together). # # Config (spec §12.1): the image takes ONLY the two bootstrap settings from the environment - -# ConnectionStrings__DispatchLog the SQL connection string (required) +# ConnectionStrings__DispatchLog the PostgreSQL (Npgsql) connection string (required) # AdminPassword first-run dashboard admin password seed (optional) -# Everything else is seeded into the SQL config table on first run and managed in the dashboard. +# Everything else is seeded into the config table on first run and managed in the dashboard. # --- Stage 1: build the React dashboard ------------------------------------------------------- FROM node:24-alpine AS ui @@ -46,7 +46,7 @@ RUN mkdir -p /var/log/dispatch /app/.dispatch-spool # if 25 is unavailable. EXPOSE 8420 8025 25 587 # Report container health from the unauthenticated /health endpoint. The dashboard is HTTPS-only with a -# self-signed cert by default, so use https + -k. start-period covers first-run schema init + SQL connect. +# self-signed cert by default, so use https + -k. start-period covers first-run schema init + DB connect. HEALTHCHECK --interval=15s --timeout=5s --start-period=40s --retries=4 \ CMD curl -fsSk https://localhost:8420/health || exit 1 # UseSystemd()/UseWindowsService() in Program.cs are no-ops here; the service runs as PID 1. diff --git a/README.md b/README.md index 89cefb53..b7b53e53 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,15 @@ Your apps / scripts → Dispatch API (port 8025) ─┘ 202 / 250 OK instantly spool directory (before any DB or (durable in-flight queue) network call) ↕ - relay_log in SQL + relay_log in PostgreSQL (after-the-fact history) ↕ Web UI (port 8420) Configure · Monitor · Debug ``` -- **`250 OK` before anything else** - Dispatch writes the message to a local spool file and acknowledges the sender immediately. SQL Server is written to only *after* the provider accepts the message. -- **The spool directory is the queue** - `.eml` files survive restarts, crashes, and SQL outages. If SQL is down, mail still flows. +- **`250 OK` before anything else** - Dispatch writes the message to a local spool file and acknowledges the sender immediately. PostgreSQL is written to only *after* the provider accepts the message. +- **The spool directory is the queue** - `.eml` files survive restarts, crashes, and database outages. If PostgreSQL is down, mail still flows. - **One place for credentials** - rotate an API key once, not in every app. - **Test before you commit** - verify provider credentials with a live relay log, or capture mail to the **Local Inbox** without delivering anything. @@ -52,7 +52,7 @@ Your apps / scripts → Dispatch API (port 8025) ─┘ - **Secure by default** - HTTPS-only dashboard, a shared TLS cert for SMTP + the API, bcrypt-hashed passwords & API keys, encrypted secrets at rest, CIDR allow-lists so it's never an open relay. - **Privacy first - no call home** - no telemetry, no analytics, no usage stats, no automatic update pings. The only outbound connections are to the mail providers you configure; updates are applied from a signed package you upload yourself. - **Observability** - unauthenticated `/health` and Prometheus `/metrics` (dashboard-port, allow-list-gated). -- **Deploy anywhere** - Windows & Linux installers (bundled SQL), a multi-arch Docker image, or a ready-to-import virtual appliance for Hyper-V, VMware, KVM & Proxmox. +- **Deploy anywhere** - Windows & Linux installers (bundled PostgreSQL), a multi-arch Docker image, or a ready-to-import virtual appliance for Hyper-V, VMware, KVM & Proxmox. ## Quickstart @@ -95,19 +95,19 @@ Full details: **[Security docs](https://docs.dispatchrelay.app/security/)**. Ple ## Building from source -Prerequisites: the **.NET 10 SDK**, **Node.js 20+**, and **Docker** (for SQL). +Prerequisites: the **.NET 10 SDK**, **Node.js 20+**, and **Docker** (for PostgreSQL). ```bash git clone https://github.com/CinderHillsDev/dispatch-platform.git cd dispatch-platform -docker compose up -d # SQL (Azure SQL Edge); schema auto-created on first run +docker compose up -d # PostgreSQL; schema auto-created on first run cd src/Dispatch.UI && npm install && npm run build && cd ../.. rm -rf src/Dispatch.Web/wwwroot && mkdir -p src/Dispatch.Web/wwwroot cp -r src/Dispatch.UI/dist/* src/Dispatch.Web/wwwroot/ ASPNETCORE_ENVIRONMENT=Development dotnet run --project src/Dispatch.Service ``` -`appsettings.Development.json` (git-ignored) needs at least the SQL connection string and an `AdminPassword`. Tests: `dotnet test` (Data integration tests run only when `DISPATCH_TEST_SQL` is set, auto-skipped otherwise). The documentation site lives in its own repo (`dispatch-docs`, published at [docs.dispatchrelay.app](https://docs.dispatchrelay.app/)); the marketing site is `dispatch-website`. +`appsettings.Development.json` (git-ignored) needs at least the PostgreSQL connection string and an `AdminPassword`. Tests: `dotnet test` (Data integration tests run only when `DISPATCH_TEST_SQL` is set, auto-skipped otherwise). The documentation site lives in its own repo (`dispatch-docs`, published at [docs.dispatchrelay.app](https://docs.dispatchrelay.app/)); the marketing site is `dispatch-website`. ## Adding a provider diff --git a/appliance/Import-DispatchAppliance.ps1 b/appliance/Import-DispatchAppliance.ps1 index 15b0a784..ccd6de28 100644 --- a/appliance/Import-DispatchAppliance.ps1 +++ b/appliance/Import-DispatchAppliance.ps1 @@ -5,7 +5,7 @@ .DESCRIPTION Creates a Generation 2 VM, copies the appliance VHDX into the chosen storage location, sets the Secure Boot template to the Microsoft UEFI Certificate Authority (required for Linux), connects a virtual - switch (optionally on a specific VLAN), and (optionally) starts it. The appliance configures SQL Server + switch (optionally on a specific VLAN), and (optionally) starts it. The appliance configures PostgreSQL + Dispatch on first boot; browse to https://:8420 and set the admin password. Run it with no networking/storage flags for a guided menu: it lists the host's virtual switches and @@ -88,7 +88,7 @@ function Select-Storage([string]$Default) { Where-Object { $_.DriveLetter -and $_.Size -gt 0 } | Sort-Object DriveLetter | ForEach-Object { Write-Host (" {0}: {1:N0} GB free of {2:N0} GB {3}" -f $_.DriveLetter, ($_.SizeRemaining / 1GB), ($_.Size / 1GB), $_.FileSystemLabel) } - Write-Host " (The appliance disk is ~6-10 GB thin-provisioned; SQL + logs grow it over time.)" + Write-Host " (The appliance disk is ~6-10 GB thin-provisioned; PostgreSQL + logs grow it over time.)" return (Read-WithDefault "VM storage folder" $Default) } @@ -177,7 +177,7 @@ try { Set-VMProcessor -VM $vm -Count $CpuCount # Linux on Gen2 needs the Microsoft UEFI CA Secure Boot template (not the default Windows one). Set-VMFirmware -VM $vm -EnableSecureBoot On -SecureBootTemplate "MicrosoftUEFICertificateAuthority" - # SQL Server needs a stable working set; disable Dynamic Memory. + # PostgreSQL needs a stable working set; disable Dynamic Memory. Set-VMMemory -VM $vm -DynamicMemoryEnabled $false # Apply a VLAN tag to the adapter if requested (access mode = single tagged VLAN). if ($VlanId -gt 0) { @@ -211,7 +211,7 @@ Write-Host ("Created '{0}' (Gen2, {1} vCPU, {2} GB, switch '{3}'{4}{5})." -f ` $Name, $CpuCount, $MemoryGB, $SwitchName, $(if ($VlanId -gt 0) { ", VLAN $VlanId" } else { "" }), $(if ($AddToCluster) { ", clustered" } else { "" })) if ($Start) { Start-VM -VM $vm - Write-Host "Started. First boot configures SQL + Dispatch (a few minutes)." + Write-Host "Started. First boot configures PostgreSQL + Dispatch (a few minutes)." } else { Write-Host "Start it with: Start-VM -Name '$Name'" } diff --git a/appliance/README.md b/appliance/README.md index f2604b6c..6a7493b7 100644 --- a/appliance/README.md +++ b/appliance/README.md @@ -1,6 +1,6 @@ # Dispatch SMTP Relay - virtual appliance -A ready-to-run **Ubuntu 24.04 LTS** virtual machine with Dispatch and SQL Server Express pre-installed. Import it, power it on, and the dashboard comes up - no .NET, SQL, or command line needed. +A ready-to-run **Ubuntu 24.04 LTS** virtual machine with Dispatch and PostgreSQL pre-installed. Import it, power it on, and the dashboard comes up - no .NET, database, or command line needed. There's a **separate download per hypervisor** - each is a single zip (unzip once) with that format's image, its README, and the import helper: @@ -10,7 +10,7 @@ There's a **separate download per hypervisor** - each is a single zip (unzip onc | **VMware** (vSphere/ESXi/Workstation/Fusion) | `dispatch-appliance-vmware` | `dispatch-appliance.ova` + README | | **KVM/libvirt & Proxmox** | `dispatch-appliance-kvm` | `dispatch-appliance.qcow2` + `import-libvirt.sh` / `import-proxmox.sh` + README | -All are **Gen2/UEFI**, ~4 GB RAM recommended (SQL Server needs ~2 GB). They boot on DHCP, but a relay your apps point at should have a **static IP** - set one after first boot (see [Static IP](#static-ip)). +All are **Gen2/UEFI**, ~4 GB RAM recommended (PostgreSQL needs ~2 GB). They boot on DHCP, but a relay your apps point at should have a **static IP** - set one after first boot (see [Static IP](#static-ip)). ## Hyper-V @@ -49,7 +49,7 @@ Creates a q35/OVMF VM, imports the disk as `scsi0`, and sets it to boot. ## First boot -On first start the appliance gives itself a **unique** SQL SA password, configures and starts SQL Server Express, and starts Dispatch (the database and schema are created automatically). This takes a couple of minutes the first time. +On first start the appliance gives itself a **unique** database password, creates the `dispatch` role and `DispatchLog` database, starts PostgreSQL, and starts Dispatch (the schema is created automatically). This takes a couple of minutes the first time. Then browse to the dashboard: @@ -90,15 +90,15 @@ Pass `-i ` to target a specific interface (see `ip a`); run `dispatch-set-i ## Maintenance (console) - Service: `systemctl status dispatch` · logs: `journalctl -u dispatch -f` and `/var/log/dispatch/`. -- SQL: `systemctl status mssql-server`. +- PostgreSQL: `systemctl status postgresql`. - Config (connection string only): `/var/lib/dispatch/appsettings.json` - everything else is in the dashboard. ## Security notes -- Every appliance generates its **own** SQL SA password, at-rest encryption key (`.dispatch-key`), dashboard TLS cert, SSH host keys, and machine-id on first boot - nothing secret is shared across downloads. +- Every appliance generates its **own** database password, at-rest encryption key (`.dispatch-key`), dashboard TLS cert, SSH host keys, and machine-id on first boot - nothing secret is shared across downloads. - The **OS login** (`ubuntu`/`dispatch`) is a known default but **must be changed on first login** - do so immediately, especially before exposing the VM beyond a trusted LAN. - The dashboard admin password is set by **you** on first login and is stored only as a bcrypt hash in the database. -- The appliance bundles **SQL Server Express**, which is free; its use is governed by Microsoft's SQL Server Express license terms. +- The appliance bundles **PostgreSQL**, which is free and open source. ## Building it yourself diff --git a/appliance/build-appliance.sh b/appliance/build-appliance.sh index 38226fbc..f37b4ff1 100755 --- a/appliance/build-appliance.sh +++ b/appliance/build-appliance.sh @@ -3,8 +3,8 @@ # Dispatch SMTP Relay - Hyper-V appliance builder (Ubuntu 24.04 LTS). # # Customizes the official Ubuntu cloud image offline with libguestfs (no Hyper-V host, no nested virt) and -# converts it to a Gen2/UEFI dynamic VHDX. SQL Server Express's binaries are baked in; each VM configures -# SQL with a unique SA password and starts Dispatch on first boot (see appliance/firstboot.sh). +# converts it to a Gen2/UEFI dynamic VHDX. PostgreSQL's binaries are baked in; each VM configures the +# database with a unique password and starts Dispatch on first boot (see appliance/firstboot.sh). # # Requires (host): libguestfs-tools, qemu-utils, curl. # Usage: @@ -73,7 +73,7 @@ echo "==> Detecting the guest kernel version (for Hyper-V integration tools, whi KVER="$(virt-ls -a "$WORK/base.img" /lib/modules 2>/dev/null | grep -E '^[0-9]' | sort -V | tail -1 || true)" echo "guest kernel: ${KVER:-}" -echo "==> Pre-downloading SQL Server Express + tools (.debs) for an offline in-guest install" +echo "==> Pre-downloading PostgreSQL (.debs) for an offline in-guest install" # Done on the host in a clean Ubuntu 24.04 container so the dependency closure matches the cloud image and # the image build itself needs no in-guest network (libguestfs passt networking is unreliable on CI). mkdir -p "$STAGE/debs" @@ -81,24 +81,14 @@ docker run --rm -e KVER="$KVER" -v "$STAGE/debs:/debs" ubuntu:24.04 bash -ec ' export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y -qq curl ca-certificates >/dev/null - # packages-microsoft-prod.deb sets up the Microsoft "prod" repo + signing key (msodbcsql18, mssql-tools18). - # Pin its SHA256 (fail-closed) so a swapped bootstrap .deb cannot install a rogue repo/key. If Microsoft - # rotates this .deb the build will fail here - re-verify the new file and update this hash. - curl -fsSL https://packages.microsoft.com/config/ubuntu/24.04/packages-microsoft-prod.deb -o /tmp/pmc.deb - echo "c13f01ac7c3001b51a9281d40dde666db5e037e05512840c319832f7852bfec4 /tmp/pmc.deb" | sha256sum -c - \ - || { echo "ERROR: packages-microsoft-prod.deb checksum mismatch - refusing to trust it"; exit 1; } - dpkg -i /tmp/pmc.deb >/dev/null - # The SQL Server engine has its own per-version feed; Ubuntu 24.04 ships SQL Server 2025 (2022 is 22.04). - curl -fsSL https://packages.microsoft.com/config/ubuntu/24.04/mssql-server-2025.list \ - -o /etc/apt/sources.list.d/mssql-server.list - apt-get update -qq + # PostgreSQL 16 ships in the default Ubuntu 24.04 repos, so no extra apt source or signing key is needed. cd /debs # Full recursive runtime dependency closure of the target packages (skip virtual/undownloadable entries). deps=$(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances \ - mssql-server mssql-tools18 unixodbc-dev | grep "^\w" | sort -u) + postgresql postgresql-contrib | grep "^\w" | sort -u) for d in $deps; do apt-get download "$d" 2>/dev/null || true; done - echo "downloaded $(ls -1 /debs/*.deb | wc -l) SQL packages" - ls /debs/mssql-server_*.deb >/dev/null # fail the build if the core package is missing + echo "downloaded $(ls -1 /debs/*.deb | wc -l) PostgreSQL packages" + ls /debs/postgresql*_*.deb >/dev/null # fail the build if the core package is missing # Hypervisor guest agents so each hypervisor manager can display the VM IP (installed offline in the guest; # NO first-boot network, so this keeps the appliance no-call-home): open-vm-tools = VMware, qemu-guest-agent @@ -120,7 +110,7 @@ echo "==> Expanding the root partition into a ${DISK_SIZE} working image" qemu-img create -f qcow2 "$WORK/disk.qcow2" "$DISK_SIZE" virt-resize --expand /dev/sda1 "$WORK/base.img" "$WORK/disk.qcow2" -echo "==> Customizing the image (SQL Server Express + Dispatch + first-boot)" +echo "==> Customizing the image (PostgreSQL + Dispatch + first-boot)" # --no-network: provisioning installs pre-downloaded .debs offline, so the appliance needs no in-guest # network (and we avoid libguestfs's passt networking, which fails on CI runners). virt-customize -a "$WORK/disk.qcow2" \ diff --git a/appliance/dispatch-firstboot.service b/appliance/dispatch-firstboot.service index 80ca8097..5de80516 100644 --- a/appliance/dispatch-firstboot.service +++ b/appliance/dispatch-firstboot.service @@ -1,8 +1,8 @@ [Unit] -Description=Dispatch appliance first-boot setup (SQL Server + per-VM secrets) +Description=Dispatch appliance first-boot setup (PostgreSQL + per-VM secrets) # Run once; the script writes the marker when it succeeds. ConditionPathExists=!/var/lib/dispatch/.appliance-firstboot-done -# Before the relay so SQL is configured and reachable before Dispatch's first start. +# Before the relay so PostgreSQL is configured and reachable before Dispatch's first start. Before=dispatch.service After=network.target diff --git a/appliance/dispatch.ovf.template b/appliance/dispatch.ovf.template index d9b09923..5e513d01 100644 --- a/appliance/dispatch.ovf.template +++ b/appliance/dispatch.ovf.template @@ -1,7 +1,7 @@ diff --git a/appliance/firstboot.sh b/appliance/firstboot.sh index 8f356feb..9f2e7cc4 100755 --- a/appliance/firstboot.sh +++ b/appliance/firstboot.sh @@ -1,43 +1,60 @@ #!/usr/bin/env bash # # Dispatch appliance - first-boot setup. Runs once (via dispatch-firstboot.service) before the Dispatch -# service: it gives this VM its own unique SQL Server SA password, configures + starts SQL Server Express, -# substitutes that password into the Dispatch connection string (a placeholder is baked into the image), and -# lets systemd start Dispatch (ordered After= this unit). The Dispatch service then creates the DispatchLog -# database and schema on first start. OS-level secrets (SSH host keys, machine-id) are regenerated by -# cloud-init / systemd; the at-rest encryption key (.dispatch-key) and dashboard TLS cert are generated by -# the service itself on first start - none of those are baked into the image, so every appliance is unique. +# service: it gives this VM its own unique PostgreSQL password, creates the dispatch role + DispatchLog +# database, starts PostgreSQL, substitutes that password into the Dispatch connection string (a placeholder +# is baked into the image), and lets systemd start Dispatch (ordered After= this unit). The Dispatch service +# then creates the DispatchLog schema on first start. OS-level secrets (SSH host keys, machine-id) are +# regenerated by cloud-init / systemd; the at-rest encryption key (.dispatch-key) and dashboard TLS cert are +# generated by the service itself on first start - none of those are baked into the image, so every appliance +# is unique. # set -euo pipefail MARKER="/var/lib/dispatch/.appliance-firstboot-done" APPSETTINGS="/var/lib/dispatch/appsettings.json" -SQLCMD="/opt/mssql-tools18/bin/sqlcmd" [ -e "$MARKER" ] && { echo "dispatch-firstboot: already done"; exit 0; } log() { echo "dispatch-firstboot: $*"; } -# 1. Unique SA password. Alnum (from a FINITE source so nothing gets SIGPIPE'd under pipefail) + a fixed -# complexity suffix → meets SQL's policy and contains no characters special to the shell, sed, or JSON. -SA_PW="$(openssl rand -base64 48 | tr -dc 'A-Za-z0-9' | cut -c1-28)Aa1!" +# 1. Unique database password. Alnum (from a FINITE source so nothing gets SIGPIPE'd under pipefail) + a fixed +# complexity suffix, containing no characters special to the shell, sed, JSON, or a SQL string literal. +DB_PW="$(openssl rand -base64 48 | tr -dc 'A-Za-z0-9' | cut -c1-28)Aa1x" -# 2. Configure SQL Server Express (EULA accepted) with that password, then enable + start it. -log "configuring SQL Server Express" -MSSQL_SA_PASSWORD="$SA_PW" MSSQL_PID="Express" ACCEPT_EULA="Y" /opt/mssql/bin/mssql-conf -n setup -systemctl enable --now mssql-server +# 2. Ensure the default cluster is initialized + started, then enable + start PostgreSQL. The Debian +# postgresql package creates a default "main" cluster on install; start it via the systemd unit. +log "configuring PostgreSQL" +systemctl enable --now postgresql -# 3. Wait for SQL to accept logins before pointing Dispatch at it. -log "waiting for SQL Server" +# 3. Wait for PostgreSQL to accept connections before creating the role/database. +log "waiting for PostgreSQL" ready=0 for _ in $(seq 1 60); do - if "$SQLCMD" -S localhost -U sa -P "$SA_PW" -C -No -Q "SELECT 1" >/dev/null 2>&1; then ready=1; break; fi + if pg_isready -h localhost -p 5432 >/dev/null 2>&1; then ready=1; break; fi sleep 2 done -[ "$ready" = 1 ] || { log "ERROR: SQL Server did not become ready"; exit 1; } +[ "$ready" = 1 ] || { log "ERROR: PostgreSQL did not become ready"; exit 1; } -# 4. Inject the generated SA password into the connection string (the image ships a __SA_PASSWORD__ placeholder). +# 4. Create the dispatch role (with the generated password) and the DispatchLog database it owns. Idempotent +# so a re-run before the marker is written does not fail on an already-created role/database. +log "creating the dispatch role and DispatchLog database" +sudo -u postgres psql -v ON_ERROR_STOP=1 <:8420. +# The appliance configures PostgreSQL + Dispatch on first boot; then browse to https://:8420. # # Usage: # sudo ./import-libvirt.sh dispatch-appliance.qcow2 [--name dispatch] [--memory 4096] [--vcpus 2] @@ -54,7 +54,7 @@ virt-install \ # qemu-guest-agent then reports the VM's IP to libvirt ("virsh domifaddr --source agent"). echo -echo "VM '$NAME' defined. First boot configures SQL + Dispatch (a few minutes)." +echo "VM '$NAME' defined. First boot configures PostgreSQL + Dispatch (a few minutes)." [ "$START" = 1 ] || echo "Start it with: virsh start $NAME" echo "Find its IP with: virsh domifaddr $NAME" echo "Then browse to https://:8420 and set the admin password." diff --git a/appliance/import-proxmox.sh b/appliance/import-proxmox.sh index 35d02e79..644f42bd 100755 --- a/appliance/import-proxmox.sh +++ b/appliance/import-proxmox.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # Import the Dispatch SMTP Relay appliance qcow2 into Proxmox VE as a UEFI (OVMF) VM. Run on the Proxmox -# host. The appliance configures SQL Server + Dispatch on first boot; then browse to https://:8420. +# host. The appliance configures PostgreSQL + Dispatch on first boot; then browse to https://:8420. # # Usage: # ./import-proxmox.sh dispatch-appliance.qcow2 [--storage local-lvm] [--bridge vmbr0] @@ -50,6 +50,6 @@ qm set "$VMID" --scsi0 "$disk" qm set "$VMID" --boot order=scsi0 echo -echo "VM $VMID ($NAME) created. First boot configures SQL + Dispatch (a few minutes)." +echo "VM $VMID ($NAME) created. First boot configures PostgreSQL + Dispatch (a few minutes)." if [ "$START" = 1 ]; then qm start "$VMID"; echo "Started."; else echo "Start it with: qm start $VMID"; fi echo "Then browse to https://:8420 and set the admin password." diff --git a/appliance/kvm-README.txt b/appliance/kvm-README.txt index bd8c6f5c..c59bb9e0 100644 --- a/appliance/kvm-README.txt +++ b/appliance/kvm-README.txt @@ -26,7 +26,7 @@ This zip contains: ------------------------------------------------------------------------------ After import ------------------------------------------------------------------------------ - * Start the VM. First boot configures SQL Server Express + Dispatch (allow a + * Start the VM. First boot configures PostgreSQL + Dispatch (allow a few minutes). The VM uses DHCP. * Browse to https://:8420 (self-signed cert - accept the warning) and SET THE ADMIN PASSWORD on the first login. diff --git a/appliance/provision.sh b/appliance/provision.sh index 0a424676..4fc68847 100755 --- a/appliance/provision.sh +++ b/appliance/provision.sh @@ -2,25 +2,21 @@ # # Dispatch appliance - in-guest provisioning, run by virt-customize during the image build (NOT at runtime). # virt-customize runs --run scripts with /bin/sh, so this is POSIX sh (no bashisms). Runs with NO network -# (libguestfs's passt networking is unreliable on CI runners): the SQL Server Express + tools packages are -# pre-downloaded on the host into /opt/stage/debs (see build-appliance.sh) and installed offline here. SQL is -# only unpacked - it's configured per-VM on first boot (appliance/firstboot.sh). +# (libguestfs's passt networking is unreliable on CI runners): the PostgreSQL packages are pre-downloaded on +# the host into /opt/stage/debs (see build-appliance.sh) and installed offline here. PostgreSQL is only +# unpacked - it's configured per-VM on first boot (appliance/firstboot.sh). # set -eu export DEBIAN_FRONTEND=noninteractive STAGE="/opt/stage" -echo "==> Accept Microsoft EULAs (debconf) for SQL Server tools" -echo "msodbcsql18 msodbcsql/ACCEPT_EULA boolean true" | debconf-set-selections -echo "mssql-tools18 mssql-tools/accept_eula boolean true" | debconf-set-selections - -echo "==> Install SQL Server Express + tools (offline from pre-downloaded .debs)" +echo "==> Install PostgreSQL (offline from pre-downloaded .debs)" dpkg -i "$STAGE"/debs/*.deb || true # may report ordering warnings; configure resolves them -# Tolerant: a guest-agent postinst can be noisy in an offline chroot (no running systemd). SQL correctness is -# hard-verified immediately below, so a nonzero configure here must not abort the build. +# Tolerant: a guest-agent postinst can be noisy in an offline chroot (no running systemd). PostgreSQL +# correctness is hard-verified immediately below, so a nonzero configure here must not abort the build. dpkg --configure -a || true -test -x /opt/mssql/bin/mssql-conf || { echo "ERROR: mssql-server did not install" >&2; exit 1; } -test -x /opt/mssql-tools18/bin/sqlcmd || { echo "ERROR: mssql-tools18 did not install" >&2; exit 1; } +ls /usr/lib/postgresql/*/bin/initdb >/dev/null 2>&1 || { echo "ERROR: postgresql did not install" >&2; exit 1; } +command -v psql >/dev/null 2>&1 || { echo "ERROR: psql (postgresql client) did not install" >&2; exit 1; } echo "==> Enable hypervisor guest agents (best-effort; each idles harmlessly on a non-matching host)" # So Hyper-V Manager / vCenter / Proxmox can show the VM's IP. The .debs are installed above; enable only the @@ -30,9 +26,9 @@ for unit in hv-kvp-daemon.service hv-vss-daemon.service hv-fcopy-daemon.service if systemctl --root=/ enable "$unit" >/dev/null 2>&1; then echo " enabled $unit"; else echo " (skip $unit - not installed)"; fi done -echo "==> Stage Dispatch (enabled, not started; SA password finalized on first boot)" +echo "==> Stage Dispatch (enabled, not started; DB password finalized on first boot)" bash "$STAGE/install.sh" --prebuilt "$STAGE/bin" --no-start \ - --sql-connection "Server=localhost;Database=DispatchLog;User Id=sa;Password=__SA_PASSWORD__;TrustServerCertificate=True;Encrypt=True" + --sql-connection "Host=localhost;Port=5432;Database=DispatchLog;Username=dispatch;Password=__DB_PASSWORD__" echo "==> Install the dispatch-set-ip helper" install -m 755 "$STAGE/dispatch-set-ip" /usr/local/sbin/dispatch-set-ip @@ -41,7 +37,7 @@ echo "==> First-boot unit + start ordering" install -m 755 -D "$STAGE/firstboot.sh" /opt/dispatch-appliance/firstboot.sh install -m 644 "$STAGE/dispatch-firstboot.service" /etc/systemd/system/dispatch-firstboot.service mkdir -p /etc/systemd/system/dispatch.service.d -# Dispatch must start only after first-boot configures SQL. +# Dispatch must start only after first-boot configures PostgreSQL. printf '[Unit]\nAfter=dispatch-firstboot.service\nRequires=dispatch-firstboot.service\n' \ > /etc/systemd/system/dispatch.service.d/10-appliance.conf # Enable both units offline. `systemctl --root=/` creates the WantedBy symlinks without a running systemd diff --git a/appliance/vmware-README.txt b/appliance/vmware-README.txt index ef8cb0b0..9f5cda9f 100644 --- a/appliance/vmware-README.txt +++ b/appliance/vmware-README.txt @@ -23,7 +23,7 @@ This zip contains: ------------------------------------------------------------------------------ After import ------------------------------------------------------------------------------ - * Power on the VM. First boot configures SQL Server Express + Dispatch (allow a + * Power on the VM. First boot configures PostgreSQL + Dispatch (allow a few minutes). The VM uses DHCP. * Browse to https://:8420 (self-signed cert - accept the warning) and SET THE ADMIN PASSWORD on the first login. diff --git a/docker-compose.yml b/docker-compose.yml index c71ba30b..e38d8fcc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,41 +1,39 @@ -# Dispatch SMTP Relay - full local stack (Dispatch + SQL), arm64-native on Apple Silicon. +# Dispatch SMTP Relay - full local stack (Dispatch + PostgreSQL), arm64-native on Apple Silicon. # # docker compose up --build # build + run everything # open https://localhost:8420 # dashboard (self-signed cert; log in with ADMIN password below) # -# Azure SQL Edge is arm64-native so it runs without emulation on Apple Silicon. It speaks the SQL -# Server wire protocol, so Dispatch talks to it unchanged. Edge is fine for local testing; production -# should use SQL Server 2025 (the installers) or Azure SQL. +# PostgreSQL backs the relay log, counters, and config. postgres:17 is multi-arch, so it runs natively on +# Apple Silicon and x86 with no emulation. services: - sql: - image: mcr.microsoft.com/azure-sql-edge:latest - container_name: dispatch-sql + db: + image: postgres:17 + container_name: dispatch-db environment: - ACCEPT_EULA: "Y" - MSSQL_SA_PASSWORD: "Dispatch_Dev_Pass123" + POSTGRES_PASSWORD: "Dispatch_Dev_Pass123" + POSTGRES_DB: "DispatchLog" ports: - - "1433:1433" + - "5432:5432" volumes: - - dispatch-sql-data:/var/opt/mssql + - dispatch-db-data:/var/lib/postgresql/data healthcheck: - # Edge has no sqlcmd; a TCP open on 1433 (bash /dev/tcp) is enough to gate Dispatch's start, - # and Dispatch retries via its restart policy if the server isn't accepting logins yet. - test: ["CMD-SHELL", "timeout 1 bash -c '-x64.exe`** | The single file. Installs SQL Server 2025 Express → the MSI → the service. | -| Windows | `Dispatch--x64.msi` | Advanced: install against an existing SQL Server (`msiexec /i Dispatch--x64.msi SQLCONN="..."`). | -| Linux | `dispatch--linux.tar.gz` | Universal (x64 + arm64). Extract, then `sudo ./install.sh --install-sql ...` (auto-detects arch). | +| Windows | **`DispatchSetup--x64.exe`** | The single file. Installs PostgreSQL → the MSI → the service. | +| Windows | `Dispatch--x64.msi` | Advanced: install against an existing PostgreSQL (`msiexec /i Dispatch--x64.msi SQLCONN="Host=localhost;Port=5432;Database=DispatchLog;Username=dispatch;Password=..."`). | +| Linux | `dispatch--linux.tar.gz` | Universal (x64 + arm64). Extract, then `sudo ./install.sh --install-postgres ...` (auto-detects arch). | | Upgrade | `dispatch-upgrade-.tar.gz` | One cross-platform package for the **dashboard self-update** (Updates page). Upload it on any appliance/Linux/Windows install. | | Any | `ghcr.io/chrismuench/dispatch-smtp-relay:` | Multi-arch (amd64+arm64) container image; pushed to GHCR by the `docker` job. | | All | `SHA256SUMS` | Verify downloads: `sha256sum -c SHA256SUMS`. | @@ -115,5 +115,5 @@ name `DISPATCH_UPDATE_SIGNING_KEY`, value = the full PEM (`-----BEGIN PRIVATE KE ## CI vs. release builds [`installers.yml`](../.github/workflows/installers.yml) validates that the installer **sources compile** -on every push that touches `installer/**` (and runs an opt-in end-to-end SQL-install smoke test). Those CI +on every push that touches `installer/**` (and runs an opt-in end-to-end PostgreSQL-install smoke test). Those CI builds are intentionally **unsigned**. Signing happens only at release time, in `release.yml`. diff --git a/docs/SPEC.md b/docs/SPEC.md index d41c13ac..e906b22a 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -60,7 +60,7 @@ The shipped implementation intentionally diverges from a few details described l - **Config provider settings** are edited per-relay via `PUT /api/relays/{id}` (named relays, §10), not a global `PUT /api/config/provider`; live in-flight counts are part of `GET /api/stats/relays`. (Affects §9.3.) - **Default source-IP allow-lists are deployment-friendly, not loopback-only.** A loopback-only default makes the dashboard unreachable on the common deployment shapes (headless servers have no local browser; containers NAT every request). So the seeded defaults are: `webui.allowed_cidrs` and `api.allowed_cidrs` = **empty (allow all)** - these surfaces are gated by the dashboard password and API keys respectively, with the CIDR list as optional hardening; `listener.allowed_cidrs` = **loopback + private ranges** (`127.0.0.1/32`, `::1/128`, `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7`) so same-host apps, private LANs and Docker networks can submit mail while the SMTP listener is **not** an open internet relay out of the box. Operators tighten any of these in the dashboard. (Affects §12.6 defaults, §17.10.) - **Provider set is larger than §8 describes.** Implemented providers: `Unconfigured` (default), `Local`, `Smtp`, `Mailgun`, `SendGrid`, `AzureCommunication`, **`AmazonSes`, `Postmark`, `Resend`, `SparkPost`, `Smtp2Go`**. The five bold ones were on Appendix A ("future") but are shipped; their credential field schemas live in `RelayProviderSchema` (Core). Anywhere §8/§10.2 says the default relay provider is `None`, read `Unconfigured`. (Affects §8, §10.2, Appendix A.) -- **Windows install is a WiX Burn bundle, not a WinForms wizard.** §15.2 describes a standalone .NET WinForms setup wizard; the actual installer is `DispatchSetup.exe`, a WiX Burn bundle (`bundle/Bundle.wxs`) that chains a thin `InstallSqlExpress.exe` launcher (runs `InstallSqlExpress.ps1` to install **SQL Server 2025 Express**, instance `DISPATCHSQL`) then the Dispatch MSI. There is no interactive multi-step UI and no certificate-generation step on Windows - automatic self-signed cert generation exists only on Linux via `install.sh --generate-cert`. (Affects §15.2, §17.2.) +- **Windows install is a WiX Burn bundle, not a WinForms wizard.** §15.2 describes a standalone .NET WinForms setup wizard; the actual installer is `DispatchSetup.exe`, a WiX Burn bundle (`bundle/Bundle.wxs`) that chains a thin `InstallPostgres.exe` launcher (runs `InstallPostgres.ps1` to silently install a bundled **PostgreSQL** server and create the `DispatchLog` database) then the Dispatch MSI. There is no interactive multi-step UI and no certificate-generation step on Windows - automatic self-signed cert generation exists only on Linux via `install.sh --generate-cert`. (Affects §15.2, §17.2.) - **The ingestion API is HTTP-only; no API-TLS config keys.** There are no `api.tls_cert_path` / `api.tls_cert_password` config keys and no API-TLS UI controls - Kestrel serves the ingestion port over plain HTTP (terminate TLS at a reverse proxy if needed). The `webui.tls_cert_path` / `webui.tls_cert_password` settings live in **appsettings.json** (§12.2), not the SQL `config` table, so they are not rows in the §12.3 table. There is also no `webui.require_auth` key - the dashboard requires auth whenever an admin password hash is set. Two config keys exist that §12.3 omits: `listener.server_name` (default `Dispatch`) and `purge.captured_retention_days` (default `7`). (Affects §9.2, §12.3.) - **Some §-pseudocode signatures are illustrative.** The internal method shapes shown in §10.7 (`RoutingEngine.ResolveAsync`) and §11.5 (provider-test service) differ from the code - e.g. `ResolveAsync` returns `ValueTask` over `IReadOnlyList` recipients, and starting a provider test is synchronous and returns a `TestRun`. Treat those listings as intent, not literal API. - **SMTP source-IP denial happens at MAIL FROM, not the greeting.** The CIDR allow-list and intake/back-pressure checks run in the mailbox filter (`CanAcceptFromAsync`); a disallowed source is refused there (and a `Denied` row is logged), rather than with a `554` at the SMTP banner. (Refines §5.3, §17.10.) @@ -98,10 +98,10 @@ A configurable pool of background workers (default: 4) each claim one spool file - **Transient failure** → increment `relay_counters.retried`; write `Retrying` row to `relay_log` only if `logging.log_retrying = true`; update `.meta` sidecar; sleep back-off; retry - **Permanent failure** → increment `relay_counters.failed`; write `Failed` row to `relay_log` (always); move file to `failed/` -SQL Server is only written to **after** the provider responds. If SQL Server is unavailable, mail still flows - the log entry will be missing from the UI until SQL recovers, but no messages are lost. +PostgreSQL is only written to **after** the provider responds. If PostgreSQL is unavailable, mail still flows - the log entry will be missing from the UI until the database recovers, but no messages are lost. -**SQL Server - Logging & Config Only** -SQL Server Express holds `relay_log` (event history, configurable suppression), `relay_counters` (daily aggregates, always written), the `config` table, and the schema version table. +**PostgreSQL - Logging & Config Only** +PostgreSQL holds `relay_log` (event history, configurable suppression), `relay_counters` (daily aggregates, always written), the `config` table, and the schema version table. **Embedded Web UI Layer** ASP.NET Core minimal API hosts a React/Vite SPA on port 8420 (default). The UI reads `relay_log` for the Message Log, reads `spool/` directory counts for the queue depth widget, and streams live events via SignalR. @@ -114,13 +114,13 @@ ASP.NET Core minimal API hosts a React/Vite SPA on port 8420 (default). The UI r | Relay dispatch | MailKit SmtpClient + provider SDKs | | **Durable queue** | **Local spool directory - `.eml` files, atomic directory moves** | | Worker wake signal | `FileSystemWatcher` + `Channel` (filename only) | -| Relay event log | SQL Server Express - `relay_log` (configurable) + `relay_counters` (always) | -| Log ORM | Microsoft.Data.SqlClient + Dapper | +| Relay event log | PostgreSQL - `relay_log` (configurable) + `relay_counters` (always) | +| Log ORM | Npgsql + Dapper | | Web host | ASP.NET Core 10 minimal API | | Real-time log push | SignalR (Microsoft.AspNetCore.SignalR) | | Web UI framework | React 18 + Vite (compiled, embedded in assembly) | -| Configuration store | SQL Server `config` table (connection string only in `appsettings.json`) | -| Structured logging | Serilog - file sink + SQL Server sink (`relay_log`) + in-memory ring buffer | +| Configuration store | PostgreSQL `config` table (connection string only in `appsettings.json`) | +| Structured logging | Serilog - file sink + PostgreSQL sink (`relay_log`) + in-memory ring buffer | | Windows service host | .NET Worker Service (IHostedService) | | Windows installer | WiX Toolset v6 (MSI) | | Linux service | systemd unit file | @@ -170,12 +170,12 @@ Dispatch/ | Microsoft.AspNetCore.SignalR | Real-time log push to the browser UI | MIT | | SendGrid (v9.x) | SendGrid upstream relay via API | MIT | | Azure.Communication.Email | Azure Communication Services relay via API | MIT | -| Microsoft.Data.SqlClient | SQL Server connection for relay_log writes | MIT | +| Npgsql | PostgreSQL connection for relay_log writes | PostgreSQL (BSD-style) | | Dapper | Lightweight ORM for relay_log inserts and UI queries | Apache 2.0 | | System.Net.Http (built-in) | Mailgun upstream relay via REST (no SDK needed) | MIT | | Microsoft.Extensions.Hosting | Worker service host / DI / lifetime management | MIT | -> **Note:** SQL Server is only ever written to *after* the provider responds. Configuration is in the SQL `config` table (loaded into memory at startup). If SQL goes down after startup, the cached config keeps services running. The web UI cannot save new settings while SQL is down, but mail continues to flow. +> **Note:** PostgreSQL is only ever written to *after* the provider responds. Configuration is in the `config` table (loaded into memory at startup). If PostgreSQL goes down after startup, the cached config keeps services running. The web UI cannot save new settings while the database is down, but mail continues to flow. --- @@ -220,7 +220,7 @@ The SMTP listener binds to all interfaces so connections arrive regardless of wh Default allow-list is `127.0.0.1/32` (localhost only). To accept from your local network, add your subnet - e.g. `192.168.1.0/24`. To accept from anywhere, set `0.0.0.0/0` (open relay - only do this with `Require AUTH` enabled). -When **Require AUTH** is enabled, Dispatch validates credentials against a username/password list stored in the `config_smtp_credentials` table in SQL Server. The web UI provides a credential manager for this list. +When **Require AUTH** is enabled, Dispatch validates credentials against a username/password list stored in the `config_smtp_credentials` table in PostgreSQL. The web UI provides a credential manager for this list. --- @@ -228,7 +228,7 @@ When **Require AUTH** is enabled, Dispatch validates credentials against a usern ### 6.1 Design Principle -The spool directory on the local filesystem is the durable queue. SQL Server is never on the critical path from SMTP receipt to `250 OK`. The flow is: +The spool directory on the local filesystem is the durable queue. PostgreSQL is never on the critical path from SMTP receipt to `250 OK`. The flow is: ``` SMTP DATA complete @@ -245,17 +245,17 @@ Worker moves file to spool/processing/ ▼ Parse with MimeKit, run RoutingEngine → call IRelayProvider │ - ├── Success ──► Write Delivered row to relay_log in SQL + ├── Success ──► Write Delivered row to relay_log in PostgreSQL │ Delete spool file │ ├── Transient ─► Update retry sidecar (.meta file) │ failure Sleep back-off interval, retry │ - └── Permanent ─► Write Failed row to relay_log in SQL + └── Permanent ─► Write Failed row to relay_log in PostgreSQL failure Move file to spool/failed/ ``` -SQL Server is written to **only after** the provider responds. If SQL Server is down, mail still flows and spool files accumulate normally - the `relay_log` insert is fire-and-forget. A failed insert is logged to the Serilog file sink (so the event is never silently lost) and the spool file is still deleted on delivery success. The UI Message Log will have gaps for the outage period; no messages are lost. +PostgreSQL is written to **only after** the provider responds. If PostgreSQL is down, mail still flows and spool files accumulate normally - the `relay_log` insert is fire-and-forget. A failed insert is logged to the Serilog file sink (so the event is never silently lost) and the spool file is still deleted on delivery success. The UI Message Log will have gaps for the outage period; no messages are lost. ### 6.2 Spool Directory Structure @@ -577,7 +577,7 @@ Files in `spool/failed/` are surfaced in the web UI Message Log with status `Fai ### 6.10 Auto-Purge and Retention -Dispatch manages data growth in two places: spool files on disk and log rows in SQL Server. +Dispatch manages data growth in two places: spool files on disk and log rows in PostgreSQL. #### Spool File Purge @@ -596,27 +596,32 @@ The `PurgeWorker` `IHostedService` runs every 6 hours (configurable) and deletes ```sql DELETE FROM relay_log -WHERE logged_at < DATEADD(DAY, -@RetentionDays, SYSUTCDATETIME()); +WHERE logged_at < now() - (@RetentionDays * INTERVAL '1 day'); ``` Deletes run in batches of 1,000 rows with a 100 ms pause between batches to avoid lock contention. -#### relay_log Purge - Size-Based Pressure +#### relay_log Purge - Size-Based Pressure (optional) -The purge worker also checks database size on every run and on service startup. If the `DispatchLog` database reaches **9.5 GB**, it deletes the oldest `relay_log` rows in batches of 500 until the database drops below **9.0 GB**: +PostgreSQL imposes **no hard database-size cap**, so size-based pressure purging is an **optional, opt-in safety valve** rather than a mandatory guardrail. It is **disabled by default**; time-based retention (above) is the primary growth control. When an operator enables it and sets a `TriggerGb` threshold, the purge worker checks the database size on every run and on service startup via `pg_database_size(current_database())`. If the `DispatchLog` database reaches the configured `TriggerGb`, it deletes the oldest `relay_log` rows in batches of 500 until the database drops below `TargetGb`: ```csharp // Priority order for size-based purge (oldest first in each phase): // 1. relay_log rows (most numerous; no raw bytes) -// SQL Server Express hard limit is 10 GB - 9.5 GB trigger leaves 500 MB buffer +// Optional feature: only runs when Purge.SizePressure.Enabled = true and a TriggerGb is set. +// PostgreSQL has no built-in size ceiling, so the operator picks a threshold that fits their disk. ``` +With the feature enabled and (for example) a 10 GB `TriggerGb` / 9.5 GB `TargetGb`: + | Database size | Behaviour | |---|---| -| < 8.0 GB | ✅ Normal | -| 8.0 – 9.0 GB | 🟡 Warning shown in web UI and logs | -| 9.0 – 9.5 GB | 🟠 Pressure purge imminent | -| ≥ 9.5 GB | 🔴 Pressure purge runs immediately | +| < 80% of TriggerGb | ✅ Normal | +| 80 - 95% of TriggerGb | 🟡 Warning shown in web UI and logs | +| 95 - 100% of TriggerGb | 🟠 Pressure purge imminent | +| ≥ TriggerGb | 🔴 Pressure purge runs immediately | + +When the feature is disabled (the default), none of these thresholds apply and `relay_log` growth is bounded only by time-based retention. #### Purge Configuration @@ -626,8 +631,9 @@ The purge worker also checks database size on every run and on service startup. "ScheduleIntervalHours": 6, "SpoolFailedRetentionDays": 30, "SizePressure": { - "TriggerGb": 9.5, - "TargetGb": 9.0 + "Enabled": false, + "TriggerGb": 10.0, + "TargetGb": 9.5 }, "Log": { "DeliveredRetentionDays": 30, @@ -636,40 +642,40 @@ The purge worker also checks database size on every run and on service startup. } ``` -### 6.11 SQL Server - Full Database Schema +### 6.11 PostgreSQL - Full Database Schema -SQL Server holds four tables. There is no `relay_queue` table. +PostgreSQL holds four tables. There is no `relay_queue` table. **relays** - named relay configurations: ```sql CREATE TABLE relays ( - id INT IDENTITY PRIMARY KEY, - name NVARCHAR(128) NOT NULL UNIQUE, - provider NVARCHAR(64) NOT NULL, -- Mailgun | SendGrid | AzureCommunication | Smtp | None - is_default BIT NOT NULL DEFAULT 0, - enabled BIT NOT NULL DEFAULT 1, - max_concurrency INT NOT NULL DEFAULT 4, -- max simultaneous dispatches; 0 = unlimited - max_message_bytes INT NOT NULL DEFAULT 0, -- 0 = use provider type default - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - updated_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(128) NOT NULL UNIQUE, + provider VARCHAR(64) NOT NULL, -- Mailgun | SendGrid | AzureCommunication | Smtp | None + is_default BOOLEAN NOT NULL DEFAULT FALSE, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + max_concurrency INT NOT NULL DEFAULT 4, -- max simultaneous dispatches; 0 = unlimited + max_message_bytes INT NOT NULL DEFAULT 0, -- 0 = use provider type default + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -- Enforce exactly one default relay -CREATE UNIQUE INDEX IX_relays_default ON relays (is_default) WHERE is_default = 1; +CREATE UNIQUE INDEX IX_relays_default ON relays (is_default) WHERE is_default = TRUE; ``` **routing_rules** - ordered routing table: ```sql CREATE TABLE routing_rules ( - id INT IDENTITY PRIMARY KEY, + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, priority INT NOT NULL UNIQUE, - name NVARCHAR(128) NOT NULL, - recipient_pattern NVARCHAR(256) NULL, - sender_pattern NVARCHAR(256) NULL, + name VARCHAR(128) NOT NULL, + recipient_pattern VARCHAR(256) NULL, + sender_pattern VARCHAR(256) NULL, relay_id INT NOT NULL REFERENCES relays(id), - enabled BIT NOT NULL DEFAULT 1, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); ``` @@ -678,48 +684,48 @@ CREATE TABLE routing_rules ( ```sql CREATE TABLE relay_log ( -- Identity - id BIGINT IDENTITY PRIMARY KEY, - logged_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - spool_id NVARCHAR(64) NOT NULL, + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + logged_at TIMESTAMPTZ NOT NULL DEFAULT now(), + spool_id VARCHAR(64) NOT NULL, -- Event - event NVARCHAR(32) NOT NULL, + event VARCHAR(32) NOT NULL, -- Received | Delivered | Retrying | Failed -- TestSent | Denied - status NVARCHAR(16) NOT NULL, + status VARCHAR(16) NOT NULL, -- OK | Error | Denied retry_attempt INT NOT NULL DEFAULT 0, -- Envelope - from_address NVARCHAR(512) NOT NULL, - from_domain NVARCHAR(255) NOT NULL, -- extracted, for indexed filtering - to_addresses NVARCHAR(MAX) NOT NULL, -- JSON array of full addresses - to_domain NVARCHAR(255) NOT NULL, -- first recipient domain, for indexed filtering - subject NVARCHAR(998) NOT NULL, + from_address VARCHAR(512) NOT NULL, + from_domain VARCHAR(255) NOT NULL, -- extracted, for indexed filtering + to_addresses TEXT NOT NULL, -- JSON array of full addresses + to_domain VARCHAR(255) NOT NULL, -- first recipient domain, for indexed filtering + subject VARCHAR(998) NOT NULL, size_bytes INT NOT NULL DEFAULT 0, -- Routing relay_id INT NULL REFERENCES relays(id), - relay_name NVARCHAR(128) NULL, -- denormalised - survives relay rename/delete + relay_name VARCHAR(128) NULL, -- denormalised - survives relay rename/delete routing_rule_id INT NULL REFERENCES routing_rules(id), - routing_rule_name NVARCHAR(128) NULL, -- denormalised - survives rule rename/delete - routing_matched BIT NOT NULL DEFAULT 0, -- 0 = fell through to default + routing_rule_name VARCHAR(128) NULL, -- denormalised - survives rule rename/delete + routing_matched BOOLEAN NOT NULL DEFAULT FALSE, -- FALSE = fell through to default -- Provider outcome - provider NVARCHAR(64) NULL, - provider_message_id NVARCHAR(256) NULL, -- ID returned by provider (e.g. Mailgun message-id) - provider_response NVARCHAR(MAX) NULL, + provider VARCHAR(64) NULL, + provider_message_id VARCHAR(256) NULL, -- ID returned by provider (e.g. Mailgun message-id) + provider_response TEXT NULL, duration_ms INT NULL, - error NVARCHAR(MAX) NULL, + error TEXT NULL, -- Ingest source - ingest_source NVARCHAR(16) NOT NULL DEFAULT 'SMTP', -- SMTP | API - source_ip NVARCHAR(64) NULL, + ingest_source VARCHAR(16) NOT NULL DEFAULT 'SMTP', -- SMTP | API + source_ip VARCHAR(64) NULL, api_key_id INT NULL REFERENCES api_keys(id), - api_key_name NVARCHAR(256) NULL, -- denormalised + api_key_name VARCHAR(256) NULL, -- denormalised -- Tags (from HTTP API o:tag or SMTP X-Dispatch-Tag header) - tags NVARCHAR(MAX) NULL -- JSON array e.g. ["welcome","onboarding"] + tags TEXT NULL -- JSON array e.g. ["welcome","onboarding"] ); -- Primary UI query: newest first; covers status/event filter + date range @@ -761,7 +767,7 @@ CREATE INDEX IX_relay_log_purge > **Why denormalise `relay_name` and `routing_rule_name`?** If a relay is renamed or a rule is deleted, the historical log still shows the name that was in effect at dispatch time. The `relay_id` / `routing_rule_id` FK columns are kept for joins when the relay still exists, but `NULL`-able so log rows survive relay deletion. -> **`from_domain` and `to_domain`** are extracted from the full address at insert time and stored as indexed columns. SQL Server cannot index a computed expression on a `NVARCHAR(MAX)` field like `to_addresses`, so extracting the first recipient domain into its own column is the only way to make domain filtering fast. +> **`from_domain` and `to_domain`** are extracted from the full address at insert time and stored as indexed columns. PostgreSQL does support functional/expression indexes over a `TEXT` field like `to_addresses`, but extracting the first recipient domain into its own denormalised column keeps the domain filters simple and index-friendly, and lets the same column drive the covering indexes below. ``` @@ -769,7 +775,7 @@ CREATE INDEX IX_relay_log_purge ```sql CREATE TABLE relay_counters ( - id INT IDENTITY PRIMARY KEY, + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, date DATE NOT NULL, relay_id INT NOT NULL REFERENCES relays(id), received BIGINT NOT NULL DEFAULT 0, @@ -783,14 +789,13 @@ CREATE TABLE relay_counters ( CREATE INDEX IX_relay_counters_date ON relay_counters (date DESC); ``` -One row per relay per day. Counters are incremented atomically using `MERGE` / upsert: +One row per relay per day. Counters are incremented atomically using an `INSERT ... ON CONFLICT` upsert: ```sql -MERGE relay_counters AS target -USING (VALUES (CAST(SYSUTCDATETIME() AS DATE), @relayId)) AS src (date, relay_id) -ON target.date = src.date AND target.relay_id = src.relay_id -WHEN MATCHED THEN UPDATE SET delivered = delivered + 1 -WHEN NOT MATCHED THEN INSERT (date, relay_id, delivered) VALUES (src.date, src.relay_id, 1); +INSERT INTO relay_counters (date, relay_id, delivered) +VALUES ((now() AT TIME ZONE 'utc')::date, @relayId, 1) +ON CONFLICT (date, relay_id) +DO UPDATE SET delivered = relay_counters.delivered + 1; ``` The Dashboard reads from `relay_counters` for all counters - never from `relay_log`. This means counters are accurate regardless of whether `relay_log` success logging is enabled or disabled. @@ -799,10 +804,10 @@ The Dashboard reads from `relay_counters` for all counters - never from `relay_l ```sql CREATE TABLE config ( - key NVARCHAR(128) NOT NULL PRIMARY KEY, - value NVARCHAR(MAX) NOT NULL, - encrypted BIT NOT NULL DEFAULT 0, - updated_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + key VARCHAR(128) NOT NULL PRIMARY KEY, + value TEXT NOT NULL, + encrypted BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); ``` @@ -810,31 +815,31 @@ CREATE TABLE config ( ```sql CREATE TABLE api_keys ( - id INT IDENTITY PRIMARY KEY, - key_id NVARCHAR(32) NOT NULL UNIQUE, -- first 12 chars of key, public - key_hash NVARCHAR(512) NOT NULL, -- bcrypt of full key - name NVARCHAR(256) NOT NULL, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - last_used_at DATETIME2 NULL, + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + key_id VARCHAR(32) NOT NULL UNIQUE, -- first 12 chars of key, public + key_hash VARCHAR(512) NOT NULL, -- bcrypt of full key + name VARCHAR(256) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used_at TIMESTAMPTZ NULL, message_count BIGINT NOT NULL DEFAULT 0, - revoked BIT NOT NULL DEFAULT 0, - revoked_at DATETIME2 NULL, + revoked BOOLEAN NOT NULL DEFAULT FALSE, + revoked_at TIMESTAMPTZ NULL, rate_limit_per_minute INT NOT NULL DEFAULT 0, -- 0 = use global default - scope NVARCHAR(64) NOT NULL DEFAULT 'send' -- reserved for future read-only keys + scope VARCHAR(64) NOT NULL DEFAULT 'send' -- reserved for future read-only keys ); -CREATE INDEX IX_api_keys_lookup ON api_keys (key_id) WHERE revoked = 0; +CREATE INDEX IX_api_keys_lookup ON api_keys (key_id) WHERE revoked = FALSE; ``` **config_smtp_credentials** - SMTP sender allow-list: ```sql CREATE TABLE config_smtp_credentials ( - id INT IDENTITY PRIMARY KEY, - username NVARCHAR(256) NOT NULL UNIQUE, - password_hash NVARCHAR(512) NOT NULL, -- bcrypt, cost factor 12 - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - last_used_at DATETIME2 NULL + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + username VARCHAR(256) NOT NULL UNIQUE, + password_hash VARCHAR(512) NOT NULL, -- bcrypt, cost factor 12 + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used_at TIMESTAMPTZ NULL ); ``` @@ -843,22 +848,22 @@ CREATE TABLE config_smtp_credentials ( ```sql CREATE TABLE schema_version ( version INT NOT NULL PRIMARY KEY, - script_name NVARCHAR(256) NOT NULL, - applied_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + script_name VARCHAR(256) NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() ); ``` -### 6.12 SQL Server - Setup +### 6.12 PostgreSQL - Setup | Item | Detail | |---|---| -| Minimum version | SQL Server Express 2019 or 2022 (free) - full SQL Server editions also supported | -| Linux support | Yes - via the official Microsoft SQL Server for Linux package | +| Minimum version | PostgreSQL 14 or newer (free, open source) | +| Linux support | Yes - via the distribution's `postgresql` package | | Database name | `DispatchLog` (renamed from `DispatchQueue` - it is a log store, not a queue) | | Connection string | Written to `appsettings.json` by the bootstrap installer; editable in the web UI | | Schema migration | Applied automatically on startup via embedded SQL scripts | -| Auth modes | Windows Auth (Windows only) or SQL Auth username/password (both platforms) | -| SQL down behaviour | Mail flows; `relay_log` inserts fire-and-forget (failed inserts logged to file sink; UI dark); `relay_counters` upserts also fire-and-forget; counters may have gaps during outage | +| Auth modes | Password auth (`scram-sha-256`) via the Npgsql connection string; local peer/trust for the bundled instance | +| DB down behaviour | Mail flows; `relay_log` inserts fire-and-forget (failed inserts logged to file sink; UI dark); `relay_counters` upserts also fire-and-forget; counters may have gaps during outage | | Setup flow | Handled entirely by the Dispatch Bootstrap Installer - see Sections 11 and 12 | @@ -1020,15 +1025,15 @@ public class ApiMessageHandler(SpoolDirectory spool, IConfigCache config) ```sql CREATE TABLE api_keys ( - id INT IDENTITY PRIMARY KEY, - key_id NVARCHAR(32) NOT NULL UNIQUE, -- public identifier, prefix of key - key_hash NVARCHAR(512) NOT NULL, -- bcrypt hash of full key - name NVARCHAR(256) NOT NULL, -- human-readable label - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - last_used_at DATETIME2 NULL, + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + key_id VARCHAR(32) NOT NULL UNIQUE, -- public identifier, prefix of key + key_hash VARCHAR(512) NOT NULL, -- bcrypt hash of full key + name VARCHAR(256) NOT NULL, -- human-readable label + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used_at TIMESTAMPTZ NULL, message_count BIGINT NOT NULL DEFAULT 0, - revoked BIT NOT NULL DEFAULT 0, - revoked_at DATETIME2 NULL, + revoked BOOLEAN NOT NULL DEFAULT FALSE, + revoked_at TIMESTAMPTZ NULL, rate_limit_per_minute INT NOT NULL DEFAULT 100 -- 0 = use global default ); ``` @@ -1078,7 +1083,7 @@ public class ApiKeyMiddleware(IApiKeyRepository keys) : IMiddleware Key verification does one SQL lookup (by `key_id` prefix, which is indexed) then a bcrypt compare. The bcrypt compare is the expensive step - cache the full key object in a short-lived memory cache (TTL: 30 seconds) keyed by the raw token to avoid bcrypt on every request under normal load. ```sql -CREATE INDEX IX_api_keys_key_id ON api_keys (key_id) WHERE revoked = 0; +CREATE INDEX IX_api_keys_key_id ON api_keys (key_id) WHERE revoked = FALSE; ``` ### 7.8 Web UI - Settings: API Keys @@ -1118,7 +1123,7 @@ A dedicated **API Keys** page under Settings: 4. Key is stored bcrypt-hashed. The UI never shows it again. -**Revoke:** clicking [✕] marks the key `revoked = 1` and sets `revoked_at`. Revoked keys are rejected immediately on the next request (no grace period). Revoked keys remain visible in the list for audit purposes and can be filtered out with a "Show revoked" toggle. +**Revoke:** clicking [✕] marks the key `revoked = TRUE` and sets `revoked_at`. Revoked keys are rejected immediately on the next request (no grace period). Revoked keys remain visible in the list for audit purposes and can be filtered out with a "Show revoked" toggle. ### 7.9 Config Keys for the API @@ -1238,11 +1243,11 @@ The Message Log is designed to handle millions of rows without degrading. Every | Date range | Date picker (from / to) | Defaults to last 24 hours on first load | | Status | Multi-select chips: Delivered · Failed · Retrying · Denied · TestSent | Default: all | | Relay | Dropdown of named relays + "Any" | Filters on `relay_name` | -| Routing rule | Dropdown of rule names + "Any" + "Default (no rule)" | Filters on `routing_rule_name`; "Default" = `routing_matched = 0` | +| Routing rule | Dropdown of rule names + "Any" + "Default (no rule)" | Filters on `routing_rule_name`; "Default" = `routing_matched = FALSE` | | Ingest source | Toggle: All · SMTP · API | Filters on `ingest_source` | | Sender domain | Text field | Exact match on `from_domain`; indexed | | Recipient domain | Text field | Exact match on `to_domain`; indexed | -| Tag | Text field | Filters using `JSON_VALUE`; slower - shown with a ⚠ indicator | +| Tag | Text field | Filters using PostgreSQL JSON operators; slower - shown with a ⚠ indicator | | Subject | Text field | Full-text search; shown with a ⚠ indicator and note that it may be slow | | API key | Dropdown (when ingest source = API) | Filters on `api_key_id` | @@ -1272,7 +1277,7 @@ Hidden by default (toggleable): Tag, Source IP, API Key, Provider Message ID. - Page size: 50 rows (configurable 25 / 50 / 100) - "Load more" button appends the next page below current results - no page numbers, no "go to page N" - Cursor is the `(logged_at, id)` tuple of the last visible row, passed as an opaque token -- Why: `OFFSET 5000 FETCH 50` requires SQL Server to scan and discard 5,000 rows first; cursor pagination is O(1) regardless of depth +- Why: `OFFSET 5000 LIMIT 50` requires PostgreSQL to scan and discard 5,000 rows first; cursor pagination is O(1) regardless of depth **No total row count displayed** - computing `COUNT(*)` on a filtered `relay_log` with millions of rows is expensive. The UI shows "Showing 50 of many" or "Showing 12" (when fewer than a page returned) - not "Page 3 of 14,829". @@ -1342,7 +1347,7 @@ The **In Flight** counter per relay is served by `GET /api/relays/concurrency` - - Provider selector dropdown (Mailgun / SendGrid / Azure Communication Services / SMTP / None) - Provider-specific fields appear dynamically based on selection -- All API keys and passwords stored AES-256 encrypted in the `config` table in SQL Server +- All API keys and passwords stored AES-256 encrypted in the `config` table in PostgreSQL - **Send Test Email** button - runs a full provider test using the current (unsaved) form values; streams results live in an inline log panel (see Section 9) - Retry policy controls (attempts, delay intervals) @@ -1470,15 +1475,15 @@ A **relay** is a named, independently configured provider instance. Each relay h ```sql CREATE TABLE relays ( - id INT IDENTITY PRIMARY KEY, - name NVARCHAR(128) NOT NULL UNIQUE, -- e.g. "Mailgun-EU", "SendGrid-Transactional" - provider NVARCHAR(64) NOT NULL, -- Mailgun | SendGrid | AzureCommunication | Smtp | None - is_default BIT NOT NULL DEFAULT 0, -- exactly one row has is_default = 1 - enabled BIT NOT NULL DEFAULT 1, - max_concurrency INT NOT NULL DEFAULT 4, -- max simultaneous dispatches; 0 = unlimited - max_message_bytes INT NOT NULL DEFAULT 0, -- 0 = use provider type default (bytes) - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - updated_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(128) NOT NULL UNIQUE, -- e.g. "Mailgun-EU", "SendGrid-Transactional" + provider VARCHAR(64) NOT NULL, -- Mailgun | SendGrid | AzureCommunication | Smtp | None + is_default BOOLEAN NOT NULL DEFAULT FALSE, -- exactly one row has is_default = TRUE + enabled BOOLEAN NOT NULL DEFAULT TRUE, + max_concurrency INT NOT NULL DEFAULT 4, -- max simultaneous dispatches; 0 = unlimited + max_message_bytes INT NOT NULL DEFAULT 0, -- 0 = use provider type default (bytes) + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -- Provider-specific settings stored as encrypted key-value pairs @@ -1490,7 +1495,7 @@ Provider credentials for each relay are stored in the existing `config` table us **The default relay:** -- Exactly one relay has `is_default = 1` at all times +- Exactly one relay has `is_default = TRUE` at all times - On first run, a single relay named "Default" is created (provider: None) - the administrator edits it to set their provider - The default relay cannot be deleted - only edited or replaced (setting another relay as default demotes the current one automatically) - The default relay is used when no routing rule matches @@ -1501,14 +1506,14 @@ Provider credentials for each relay are stored in the existing `config` table us ```sql CREATE TABLE routing_rules ( - id INT IDENTITY PRIMARY KEY, + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, priority INT NOT NULL, -- lower number = evaluated first - name NVARCHAR(128) NOT NULL, -- human label, e.g. "Acme Corp → Mailgun EU" - recipient_pattern NVARCHAR(256) NULL, -- NULL = match any recipient - sender_pattern NVARCHAR(256) NULL, -- NULL = match any sender + name VARCHAR(128) NOT NULL, -- human label, e.g. "Acme Corp → Mailgun EU" + recipient_pattern VARCHAR(256) NULL, -- NULL = match any recipient + sender_pattern VARCHAR(256) NULL, -- NULL = match any sender relay_id INT NOT NULL REFERENCES relays(id), - enabled BIT NOT NULL DEFAULT 1, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE UNIQUE INDEX IX_routing_rules_priority ON routing_rules (priority); @@ -1716,8 +1721,8 @@ A **Routing Rules** page under Settings, below Relays: The `relay_log` table gains a `relay_id` column so every log entry records which named relay was used: ```sql -ALTER TABLE relay_log ADD relay_id INT NULL REFERENCES relays(id); -ALTER TABLE relay_log ADD relay_name NVARCHAR(128) NULL; -- denormalised for display after relay deleted +ALTER TABLE relay_log ADD COLUMN relay_id INT NULL REFERENCES relays(id); +ALTER TABLE relay_log ADD COLUMN relay_name VARCHAR(128) NULL; -- denormalised for display after relay deleted ``` The Message Log UI adds a **Relay** column so administrators can see at a glance which relay handled each message. @@ -1926,7 +1931,7 @@ public class ProviderTestService(IRelayProviderFactory factory, } ``` -**Completed test runs** are held in memory for 30 minutes then evicted (a `Timer`-based cleanup pass). They are never persisted to SQL Server - test runs are ephemeral UI state only. +**Completed test runs** are held in memory for 30 minutes then evicted (a `Timer`-based cleanup pass). They are never persisted to PostgreSQL - test runs are ephemeral UI state only. ### 11.6 Per-Provider Log Detail @@ -1961,22 +1966,22 @@ The test deliberately exercises the full code path a live message would take, so **`appsettings.json` contains exactly two things: the database connection string and the web UI TLS certificate path.** -The TLS cert path is the one exception to the "everything in SQL" rule - ASP.NET Core needs it to start the HTTPS listener before the SQL connection is established, so it cannot live in the database. Everything else is in SQL. +The TLS cert path is the one exception to the "everything in the database" rule - ASP.NET Core needs it to start the HTTPS listener before the PostgreSQL connection is established, so it cannot live in the database. Everything else is in PostgreSQL. -Everything else - listener ports, SMTP auth credentials, spool directory, worker count, retry policy, provider selection, API keys, purge settings, web UI port and password - is stored in a `config` table in SQL Server and managed through the web UI. This means: +Everything else - listener ports, SMTP auth credentials, spool directory, worker count, retry policy, provider selection, API keys, purge settings, web UI port and password - is stored in a `config` table in PostgreSQL and managed through the web UI. This means: - Minimal config file - only two fields, both written by the bootstrap installer - Settings changes from the web UI are live immediately - no file reload, no restart -- All sensitive values (API keys, passwords) are encrypted at rest in SQL with a machine-specific key -- Upgrading never has a "config file migration" problem - SQL schema migrations handle it -- The service cannot start without SQL, but once running, all configuration is authoritative from the database +- All sensitive values (API keys, passwords) are encrypted at rest in the database with a machine-specific key +- Upgrading never has a "config file migration" problem - database schema migrations handle it +- The service cannot start without PostgreSQL, but once running, all configuration is authoritative from the database ### 12.2 `appsettings.json` - Connection String and TLS Cert ```json { "ConnectionStrings": { - "DispatchLog": "Server=localhost\\SQLEXPRESS;Database=DispatchLog;Integrated Security=true;" + "DispatchLog": "Host=localhost;Port=5432;Database=DispatchLog;Username=dispatch;Password=" }, "WebUi": { "TlsCertPath": "/etc/dispatch/cert.pfx", @@ -1985,7 +1990,7 @@ Everything else - listener ports, SMTP auth credentials, spool directory, worker } ``` -The `WebUi.TlsCertPath` and `WebUi.TlsCertPassword` fields are stored in `appsettings.json` - not in the SQL `config` table - because ASP.NET Core needs them to start the HTTPS listener before SQL is reachable. The cert password is encrypted using the same machine-specific key as all other secrets. All other settings remain in SQL. +The `WebUi.TlsCertPath` and `WebUi.TlsCertPassword` fields are stored in `appsettings.json` - not in the `config` table - because ASP.NET Core needs them to start the HTTPS listener before PostgreSQL is reachable. The cert password is encrypted using the same machine-specific key as all other secrets. All other settings remain in PostgreSQL. **File locations:** @@ -2000,14 +2005,14 @@ All application settings live in a `config` table with a simple key-value struct ```sql CREATE TABLE config ( - key NVARCHAR(128) NOT NULL PRIMARY KEY, - value NVARCHAR(MAX) NOT NULL, - encrypted BIT NOT NULL DEFAULT 0, - updated_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + key VARCHAR(128) NOT NULL PRIMARY KEY, + value TEXT NOT NULL, + encrypted BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); ``` -Values marked `encrypted = 1` are stored AES-256-GCM encrypted using a random 256-bit key persisted in a portable `.dispatch-key` file (access-restricted: mode 600 on Unix; ACL-locked on Windows, in the ProgramData data dir whose folder ACL the installer also restricts). The key is portable, so a DB backup restores on a different machine when the key file is restored too. If no writable key directory is available it falls back to a PBKDF2 machine-derived key (weaker). Earlier Windows builds used DPAPI (LocalMachine); those legacy values are still readable and migrate to AES on the next save. The encryption/decryption is transparent - the `ConfigRepository` handles it on read/write. +Values marked `encrypted = TRUE` are stored AES-256-GCM encrypted using a random 256-bit key persisted in a portable `.dispatch-key` file (access-restricted: mode 600 on Unix; ACL-locked on Windows, in the ProgramData data dir whose folder ACL the installer also restricts). The key is portable, so a DB backup restores on a different machine when the key file is restored too. If no writable key directory is available it falls back to a PBKDF2 machine-derived key (weaker). Earlier Windows builds used DPAPI (LocalMachine); those legacy values are still readable and migrate to AES on the next save. The encryption/decryption is transparent - the `ConfigRepository` handles it on read/write. **Config keys:** @@ -2056,8 +2061,9 @@ Values marked `encrypted = 1` are stored AES-256-GCM encrypted using a random 25 | `purge.enabled` | bool | `true` | No | | `purge.schedule_interval_hours` | int | `6` | No | | `purge.spool_failed_retention_days` | int | `30` | No | -| `purge.size_trigger_gb` | float | `9.5` | No | -| `purge.size_target_gb` | float | `9.0` | No | +| `purge.size_pressure_enabled` | bool | `false` | No | +| `purge.size_trigger_gb` | float | `10.0` | No | +| `purge.size_target_gb` | float | `9.5` | No | | `logging.log_delivered` | bool | `true` | No | | `logging.log_retrying` | bool | `true` | No | | `logging.log_denied` | bool | `true` | No | @@ -2070,11 +2076,11 @@ SMTP sender credentials (the username/password pairs checked when `listener.requ ```sql CREATE TABLE config_smtp_credentials ( - id INT IDENTITY PRIMARY KEY, - username NVARCHAR(256) NOT NULL UNIQUE, - password_hash NVARCHAR(512) NOT NULL, -- bcrypt, cost factor 12 - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - last_used_at DATETIME2 NULL + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + username VARCHAR(256) NOT NULL UNIQUE, + password_hash VARCHAR(512) NOT NULL, -- bcrypt, cost factor 12 + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used_at TIMESTAMPTZ NULL ); ``` @@ -2082,7 +2088,7 @@ CREATE TABLE config_smtp_credentials ( ### 12.5 Reading Config at Runtime -At startup Dispatch loads the full config table into a `ConfigCache` in memory - one SQL query, all keys. This cache is the source of truth for `IRelayProvider`, `SpoolWorkerPool`, `PurgeWorker`, and all other consumers: +At startup Dispatch loads the full config table into a `ConfigCache` in memory - one query, all keys. This cache is the source of truth for `IRelayProvider`, `SpoolWorkerPool`, `PurgeWorker`, and all other consumers: ```csharp public class ConfigCache(IConfigRepository repo) @@ -2100,7 +2106,7 @@ public class ConfigCache(IConfigRepository repo) ``` When the web UI saves a setting via `PUT /api/config/{section}`, the endpoint: -1. Writes the updated key(s) to SQL +1. Writes the updated key(s) to PostgreSQL 2. Calls `ConfigCache.LoadAsync()` to refresh the in-memory cache 3. Returns `200 OK` @@ -2110,13 +2116,13 @@ Workers and services always read from `ConfigCache` - never from `IOptionsMonito When Dispatch starts and the `config` table is empty (first run after schema migration), it populates all keys with their default values. This means a freshly installed instance is immediately usable with sensible defaults - the administrator only needs to configure the relay provider. -### 12.7 Config and SQL Outage +### 12.7 Config and Database Outage -The `ConfigCache` is loaded at startup. If SQL goes down after startup, the in-memory cache remains valid and all services continue with the last-known settings. Config writes (from the web UI) fail with a visible error while SQL is down - the user is told to try again once SQL recovers. Mail flow is unaffected. +The `ConfigCache` is loaded at startup. If PostgreSQL goes down after startup, the in-memory cache remains valid and all services continue with the last-known settings. Config writes (from the web UI) fail with a visible error while the database is down - the user is told to try again once PostgreSQL recovers. Mail flow is unaffected. -### 12.8 Startup Failure - SQL Unavailable +### 12.8 Startup Failure - Database Unavailable -If SQL Server cannot be reached at startup, Dispatch cannot load its config and cannot start. It logs a clear error to the file sink and exits with a non-zero code. The systemd unit (Linux) and Windows Service manager will retry on the configured restart interval. This is the correct behaviour - without config, there is nothing safe to do. +If PostgreSQL cannot be reached at startup, Dispatch cannot load its config and cannot start. It logs a clear error to the file sink and exits with a non-zero code. The systemd unit (Linux) and Windows Service manager will retry on the configured restart interval. This is the correct behaviour - without config, there is nothing safe to do. The connection string in `appsettings.json` is the only dependency Dispatch has at startup. If that file is missing or the connection string is wrong, the bootstrap should be re-run. @@ -2130,7 +2136,7 @@ Dispatch stores relay event logs in **two places** with different purposes: | Store | What | Purpose | |---|---|---| -| `relay_log` (SQL Server) | One row per relay lifecycle event | Web UI Message Log - searchable, filterable, auto-purged | +| `relay_log` (PostgreSQL) | One row per relay lifecycle event | Web UI Message Log - searchable, filterable, auto-purged | | File (rolling daily) | Full Serilog structured output | Diagnostics, crash analysis, support - kept on disk | | Console | Full Serilog output | Foreground / Docker / journalctl live tail | | InMemoryRingBuffer | Last 1,000 events | Real-time SignalR push to the Dashboard | @@ -2479,7 +2485,7 @@ GET /health ```json { "status": "degraded", - "message": "SQL Server unavailable - mail flow unaffected; UI log unavailable", + "message": "PostgreSQL unavailable - mail flow unaffected; UI log unavailable", ... } ``` @@ -2516,7 +2522,7 @@ Windows installation uses two separate artefacts that run in sequence: | Artefact | Role | |---|---| -| `DispatchSetup.exe` | **Bootstrap installer** - handles SQL Server detection/install and database setup, then hands off to the MSI | +| `DispatchSetup.exe` | **Bootstrap installer** - handles PostgreSQL detection/install and database setup, then hands off to the MSI | | `Dispatch-{version}-x64.msi` | **Application installer** - installs binaries, registers the Windows Service, creates shortcuts | Users always run `DispatchSetup.exe`. The MSI is never run directly in normal usage. @@ -2537,10 +2543,10 @@ The bootstrap is a standalone .NET 10 WinForms application (single-file, self-co │ ▼ ┌─────────────────────────────────────────────┐ -│ SQL Server Setup │ +│ PostgreSQL Setup │ │ │ -│ ● Install SQL Server Express (recommended) │ -│ ○ Connect to an existing SQL Server │ +│ ● Install PostgreSQL (recommended) │ +│ ○ Connect to an existing PostgreSQL │ │ [Next] │ └─────────────────────────────────────────────┘ │ @@ -2551,10 +2557,10 @@ The bootstrap is a standalone .NET 10 WinForms application (single-file, self-co │ │ ▼ ▼ ┌──────────┐ ┌──────────────────────────────┐ -│ Download │ │ Connection Details │ -│ & silent │ │ Server: [____________] │ -│ install │ │ Auth: ● Windows Auth │ -│ SQL Expr │ │ ○ SQL Auth │ +│ Silent │ │ Connection Details │ +│ install │ │ Host: [____________] │ +│ bundled │ │ Port: [5432_______] │ +│ Postgres │ │ Database: [DispatchLog__] │ └──────────┘ │ Username: [____________] │ │ │ Password: [____________] │ │ │ [Test Connection]│ @@ -2591,49 +2597,45 @@ The bootstrap is a standalone .NET 10 WinForms application (single-file, self-co └─────────────────────────────────────────────┘ ``` -#### SQL Server Detection Logic +#### PostgreSQL Detection Logic On launch, the bootstrap silently attempts to connect to the following candidates in order: -1. `localhost\SQLEXPRESS` - default SQL Server Express named instance -2. `localhost` - default SQL Server instance -3. Any instance advertised by SQL Server Browser on the local machine (via `SqlDataSourceEnumerator`) +1. `localhost:5432` - a PostgreSQL server on the default port +2. Any PostgreSQL Windows service already registered on the local machine (enumerated via the Service Control Manager) -If a reachable instance is found, the wizard pre-selects **Connect to an existing SQL Server** and pre-fills the server name. The user can confirm or change it. +If a reachable instance is found, the wizard pre-selects **Connect to an existing PostgreSQL** and pre-fills the host/port. The user can confirm or change it. -If no instance is found, the wizard pre-selects **Install SQL Server Express (recommended)**. +If no instance is found, the wizard pre-selects **Install PostgreSQL (recommended)**. -#### SQL Server Express Silent Install +#### PostgreSQL Silent Install -When the user chooses to install SQL Server Express, the bootstrap: +When the user chooses to install PostgreSQL, the bootstrap installs a **PostgreSQL server bundled with the setup package** - no download is required at install time. The bootstrap: -1. Downloads `SQLEXPR_x64_ENU.exe` from the official Microsoft URL (shown to the user with progress bar) -2. Verifies the SHA-256 checksum against a value hardcoded in the bootstrap before running it -3. Launches the installer silently: +1. Verifies the SHA-256 checksum of the bundled PostgreSQL installer against a value hardcoded in the bootstrap before running it +2. Launches the installer silently (unattended), initialising a data directory, setting the superuser password, and registering the Windows service on port `5432`: ``` -SQLEXPR_x64_ENU.exe /Q /ACTION=Install /FEATURES=SQLEngine - /INSTANCENAME=SQLEXPRESS - /SQLSVCACCOUNT="NT AUTHORITY\NETWORK SERVICE" - /SQLSYSADMINACCOUNTS="BUILTIN\Administrators" - /TCPENABLED=1 - /NPENABLED=0 - /IACCEPTSQLSERVERLICENSETERMS +postgresql-installer.exe --mode unattended --unattendedmodeui none \ + --superpassword "" --serverport 5432 \ + --servicename DispatchPostgres --datadir "C:\ProgramData\Dispatch\pgdata" ``` -4. Waits for completion and verifies the service is running before proceeding +3. Waits for completion and verifies the service is running before proceeding +4. Creates the `DispatchLog` database and a least-privilege `dispatch` role (see below) #### Connection Details Screen -When the user chooses **Connect to an existing SQL Server**: +When the user chooses **Connect to an existing PostgreSQL**: -- **Server** - free-text field, accepts `hostname`, `hostname\instance`, `hostname,port` -- **Auth mode** - Windows Auth (default) or SQL Auth -- **Username / Password** - shown only when SQL Auth is selected +- **Host** - free-text field, accepts `hostname` or an IP address +- **Port** - defaults to `5432` +- **Database** - defaults to `DispatchLog` +- **Username / Password** - the PostgreSQL role and password Dispatch connects with - **Test Connection** button - attempts `SELECT 1` and reports success or the exact error message - The user cannot proceed until Test Connection succeeds -The bootstrap needs `db_creator` rights on the target instance (to create the `DispatchLog` database). After initial setup, Dispatch only needs `db_datawriter` + `db_datareader` on that database. The bootstrap creates a dedicated `dispatch` SQL login with minimal rights after database creation (when SQL Auth is used). +The bootstrap needs `CREATEDB` rights on the target server (to create the `DispatchLog` database). After initial setup, Dispatch only needs `SELECT` / `INSERT` / `UPDATE` / `DELETE` on that database's tables. The bootstrap creates a dedicated least-privilege `dispatch` PostgreSQL role after database creation. #### Database Initialisation @@ -2658,7 +2660,7 @@ The MSI finds the pre-written `appsettings.json` in `ProgramData` and does not t ### 15.3 MSI (`Dispatch-{version}-x64.msi`) -Authored with WiX Toolset v6. Targets x64 Windows 10 / Server 2019 and later. The MSI assumes SQL Server and the database are already configured (by the bootstrap) - it does not interact with SQL Server. +Authored with WiX Toolset v6. Targets x64 Windows 10 / Server 2019 and later. The MSI assumes PostgreSQL and the database are already configured (by the bootstrap) - it does not interact with PostgreSQL. **MSI Actions:** @@ -2792,7 +2794,7 @@ Bootstrap detects existing install │ New: v1.1.0 │ │ │ │ ● Upgrade (recommended) │ -│ ○ Change SQL Server connection │ +│ ○ Change PostgreSQL connection │ │ │ │ [Upgrade] │ └─────────────────────────────────────────┘ @@ -2830,7 +2832,7 @@ Before stopping the service, the bootstrap calls `POST /api/service/drain` (or r #### 11.4.3 Database Schema Migration -The bootstrap applies schema migrations as part of every upgrade, before the MSI runs. Migrations are idempotent SQL scripts embedded in the bootstrap EXE, numbered sequentially: +The bootstrap applies schema migrations as part of every upgrade, before the MSI runs. Migrations are idempotent PostgreSQL scripts embedded in the bootstrap EXE, numbered sequentially: ``` Dispatch.Bootstrap.Windows/ @@ -2846,8 +2848,8 @@ A `schema_version` table tracks which scripts have been applied: ```sql CREATE TABLE IF NOT EXISTS schema_version ( version INT NOT NULL PRIMARY KEY, - script_name NVARCHAR(256) NOT NULL, - applied_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + script_name VARCHAR(256) NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() ); ``` @@ -2861,7 +2863,7 @@ Each migration script is wrapped in a transaction with a `ROLLBACK` on error, so |---|---| | `appsettings.json` | **Preserved** - contains only the connection string; MSI never overwrites it | | Encryption keys / secrets | **Preserved** - machine-specific key derived from OS; no key material stored in files | -| Queue data in SQL Server | **Preserved** - only additive schema changes; no destructive migrations | +| Queue data in PostgreSQL | **Preserved** - only additive schema changes; no destructive migrations | | Delivered / failed message history | **Preserved** - purge policy unchanged | | Firewall rules | **Recreated** - old rules removed, new rules added by the new MSI (same ports unless overridden) | | Windows Service registration | **Preserved** - `ServiceInstall` uses the same service name; WiX handles the update | @@ -2871,7 +2873,7 @@ Each migration script is wrapped in a transaction with a `ROLLBACK` on error, so The MSI uses Windows Installer's built-in transactional rollback. If the MSI step fails mid-install, Windows Installer reverts file copies and registry changes automatically. The bootstrap re-starts the previous service version if the MSI rolls back. -Database schema migrations are **not** automatically rolled back (SQL Server DDL is not transactional across all statement types). If a migration fails: +Database schema migrations are **not** automatically rolled back across the whole upgrade (each migration runs in its own transaction, but earlier migrations that already committed stay applied). If a migration fails: 1. The bootstrap logs the exact failed script and SQL error 2. The service is not started @@ -2884,22 +2886,22 @@ Database schema migrations are **not** automatically rolled back (SQL Server DDL #### 11.4.6 Silent Upgrade ```bat -:: Upgrade non-interactively - reads SQL Server connection from appsettings.json +:: Upgrade non-interactively - reads PostgreSQL connection from appsettings.json DispatchSetup-1.1.0-x64.exe --silent --upgrade :: Upgrade with explicit connection (overrides stored connection string) -DispatchSetup-1.1.0-x64.exe --silent --upgrade --server "DBSERVER\SQLEXPRESS" --auth windows +DispatchSetup-1.1.0-x64.exe --silent --upgrade --host DBSERVER --port 5432 --user dispatch --password

``` -The `--upgrade` flag tells the bootstrap to skip the SQL Server setup wizard and go straight to migration + MSI. Without it in silent mode, the bootstrap will still detect an existing install and upgrade automatically, but `--upgrade` makes the intent explicit and skips any interactive prompts. +The `--upgrade` flag tells the bootstrap to skip the PostgreSQL setup wizard and go straight to migration + MSI. Without it in silent mode, the bootstrap will still detect an existing install and upgrade automatically, but `--upgrade` makes the intent explicit and skips any interactive prompts. ### 15.5 Silent / Unattended Install -For environments where SQL Server is already present, the full install can be scripted: +For environments where PostgreSQL is already present, the full install can be scripted: ```bat -:: Step 1 - run bootstrap in unattended mode (skips wizard, uses existing SQL Server) -DispatchSetup.exe --silent --server "DBSERVER\SQLEXPRESS" --auth windows +:: Step 1 - run bootstrap in unattended mode (skips wizard, uses existing PostgreSQL) +DispatchSetup.exe --silent --host DBSERVER --port 5432 --user dispatch --password

:: Step 2 - the bootstrap launches the MSI automatically when done :: Or launch the MSI directly after a manual bootstrap run: @@ -2911,11 +2913,11 @@ The bootstrap accepts the following CLI flags for unattended mode: | Flag | Description | |---|---| | `--silent` | Skip wizard UI; exit with code 0 on success, non-zero on failure | -| `--upgrade` | Explicit upgrade mode - skip SQL Server wizard, drain queue, migrate, install MSI | -| `--server ` | SQL Server instance to connect to | -| `--auth windows` | Use Windows Authentication (default) | -| `--auth sql --user --password

` | Use SQL Authentication | -| `--install-sqlexpress` | Download and install SQL Server Express instead of connecting | +| `--upgrade` | Explicit upgrade mode - skip PostgreSQL wizard, drain queue, migrate, install MSI | +| `--host ` | PostgreSQL host to connect to | +| `--port ` | PostgreSQL port (default 5432) | +| `--user --password

` | PostgreSQL role and password Dispatch connects with | +| `--install-postgres` | Install the bundled PostgreSQL server instead of connecting | | `--no-launch-msi` | Run database setup only; do not launch the MSI | | `--smtp-port ` | Override SMTP firewall rule port passed to MSI (default 25) | | `--submission-port ` | Override submission firewall rule port passed to MSI (default 587) | @@ -2928,11 +2930,11 @@ The bootstrap accepts the following CLI flags for unattended mode: ### 16.1 Overview - Two-Part Delivery -Linux installation mirrors the Windows approach: a bootstrap script handles SQL Server detection/install and database setup, then installs the Dispatch service. +Linux installation mirrors the Windows approach: a bootstrap script handles PostgreSQL detection/install and database setup, then installs the Dispatch service. | Artefact | Role | |---|---| -| `install.sh` | **Bootstrap script** - interactive wizard in the terminal; handles SQL Server and hands off to service install | +| `install.sh` | **Bootstrap script** - interactive wizard in the terminal; handles PostgreSQL and hands off to service install | | `Dispatch-{version}-linux-x64.tar.gz` | Application binary + default config; extracted by `install.sh` | ### 16.2 Bootstrap Script (`install.sh`) @@ -2946,40 +2948,37 @@ The script is a bash wizard that runs interactively in the terminal. It requires Dispatch Setup ============================================= -Checking for SQL Server... +Checking for PostgreSQL... - No SQL Server instance found on this machine. + No PostgreSQL instance found on this machine. -SQL Server Setup - 1) Install SQL Server Express (recommended, free) - 2) Connect to an existing SQL Server +PostgreSQL Setup + 1) Install PostgreSQL (recommended, free) + 2) Connect to an existing PostgreSQL Choice [1]: _ ``` -**If option 1 - Install SQL Server Express:** +**If option 1 - Install PostgreSQL:** ``` -Downloading Microsoft SQL Server 2025 Express... -Adding Microsoft package repository... -Installing mssql-server... -Running initial configuration (SA password required)... - SA Password: ________ +Adding the distribution's PostgreSQL package... +Installing postgresql... +Running initial configuration (dispatch role password required)... + DB Password: ________ Confirm: ________ -Starting SQL Server service... ✓ -SQL Server installed and running. +Starting PostgreSQL service... ✓ +PostgreSQL installed and running. ``` **If option 2 - Connect to existing:** ``` -Server (e.g. myserver or myserver\instance): _ -Authentication: - 1) Windows/Kerberos - 2) SQL Auth (username + password) -Choice [2]: _ +Host (e.g. localhost or 10.0.0.5): _ +Port [5432]: _ +Database [DispatchLog]: _ Username: _ Password: _ -Testing connection... ✓ Connected to SQL Server 2025 Express +Testing connection... ✓ Connected to PostgreSQL ``` **Then for both paths:** @@ -3012,49 +3011,43 @@ Installing Dispatch service... ============================================= ``` -#### SQL Server Detection +#### PostgreSQL Detection -The script checks for a running `sqlservr` process and attempts `sqlcmd -S localhost -Q "SELECT 1"`. If successful it pre-selects option 2 and displays the detected instance name. +The script checks for a running `postgres` process and attempts `psql -h localhost -c "SELECT 1"`. If successful it pre-selects option 2 and displays the detected host/port. -#### SQL Server Express Silent Install (Linux) +#### PostgreSQL Silent Install (Linux) ```bash -# Add Microsoft package repository -curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor \ - -o /usr/share/keyrings/microsoft-prod.gpg -curl -fsSL https://packages.microsoft.com/config/ubuntu/24.04/mssql-server-2025.list \ - -o /etc/apt/sources.list.d/mssql-server-2025.list - +# Install the distribution's PostgreSQL package (Debian/Ubuntu shown; RHEL/Fedora uses dnf) apt-get update -apt-get install -y mssql-server +apt-get install -y postgresql -# Run mssql-conf setup in non-interactive mode -MSSQL_SA_PASSWORD="$sa_password" \ -MSSQL_PID=Express \ -/opt/mssql/bin/mssql-conf -n setup accept-eula +systemctl enable postgresql +systemctl start postgresql -systemctl enable mssql-server -systemctl start mssql-server +# Create the database and a least-privilege role (run as the postgres superuser) +sudo -u postgres psql -c "CREATE ROLE dispatch LOGIN PASSWORD '$db_password';" +sudo -u postgres psql -c "CREATE DATABASE \"DispatchLog\" OWNER dispatch;" ``` -The SA password is prompted interactively and validated for SQL Server complexity requirements before being used. It is never written to disk. +The `dispatch` role password is prompted interactively before being used. It is never written to disk (only into the mode-600 `appsettings.json` connection string). #### Database Initialisation After a successful connection: -1. `sqlcmd` creates `DispatchLog` if it does not exist +1. `psql` creates `DispatchLog` if it does not exist 2. Embedded schema SQL scripts are applied in order -3. A dedicated `dispatch` SQL login is created with `db_datawriter` + `db_datareader` rights only -4. The SQL Server connection string is written to `/etc/dispatch/appsettings.json` (connection string only - no other settings) +3. A dedicated least-privilege `dispatch` PostgreSQL role is created with only `SELECT` / `INSERT` / `UPDATE` / `DELETE` on the Dispatch tables +4. The PostgreSQL connection string is written to `/etc/dispatch/appsettings.json` (connection string only - no other settings) ### 16.3 systemd Unit ```ini [Unit] Description=Dispatch SMTP Relay SMTP Relay Server -After=network.target mssql-server.service -Wants=mssql-server.service +After=network.target postgresql.service +Wants=postgresql.service [Service] Type=notify @@ -3073,24 +3066,24 @@ SyslogIdentifier=dispatch WantedBy=multi-user.target ``` -`After=mssql-server.service` ensures SQL Server is running before Dispatch starts. If the user connected to a remote SQL Server the `Wants`/`After` lines for `mssql-server.service` are omitted by the install script. +`After=postgresql.service` ensures PostgreSQL is running before Dispatch starts. If the user connected to a remote PostgreSQL the `Wants`/`After` lines for `postgresql.service` are omitted by the install script. ### 16.4 Unattended / CI Install ```bash -# Install SQL Server Express and Dispatch non-interactively +# Install PostgreSQL and Dispatch non-interactively curl -sSL https://github.com/yourorg/dispatch/releases/latest/download/install.sh | \ sudo bash -s -- \ - --install-sqlexpress \ - --sa-password "Str0ngP@ssword!" \ + --install-postgres \ + --db-password "Str0ngP@ssword!" \ --silent # Or connect to an existing instance curl -sSL .../install.sh | \ sudo bash -s -- \ - --server "10.0.0.5" \ - --auth sql \ - --user dotrelay_setup \ + --host "10.0.0.5" \ + --port 5432 \ + --user dispatch_setup \ --password "SetupP@ss!" \ --silent ``` @@ -3107,7 +3100,7 @@ sudo journalctl -u dispatch -f # Restart service sudo systemctl restart dispatch -# Re-run setup (e.g. to change SQL Server connection) +# Re-run setup (e.g. to change PostgreSQL connection) sudo /usr/local/bin/dispatch-setup ``` @@ -3128,7 +3121,7 @@ Existing installation detected: v1.0.0 New version: v1.1.0 1) Upgrade to v1.1.0 (recommended) - 2) Change SQL Server connection and upgrade + 2) Change PostgreSQL connection and upgrade 3) Exit Choice [1]: _ @@ -3153,7 +3146,7 @@ Before stopping the service, `install.sh` calls `curl -sk -X POST https://localh #### 12.6.3 Database Schema Migration -Identical migration logic to the Windows bootstrap (see Section 11.4.3). The `install.sh` script extracts and runs the numbered SQL scripts embedded in the tarball against the configured SQL Server instance using `sqlcmd`. The `schema_version` table ensures only unapplied scripts run. +Identical migration logic to the Windows bootstrap (see Section 11.4.3). The `install.sh` script extracts and runs the numbered SQL scripts embedded in the tarball against the configured PostgreSQL instance using `psql`. The `schema_version` table ensures only unapplied scripts run. #### 12.6.4 Configuration Preservation @@ -3161,7 +3154,7 @@ Identical migration logic to the Windows bootstrap (see Section 11.4.3). The `in |---|---| | `/etc/dispatch/appsettings.json` | **Preserved** - contains only the connection string; `install.sh` never overwrites it | | Encryption keys / secrets | **Preserved** - derived from `/etc/machine-id` at runtime; no key material stored | -| Queue data in SQL Server | **Preserved** - additive-only migrations | +| Queue data in PostgreSQL | **Preserved** - additive-only migrations | | Log files in `/var/log/dispatch/` | **Preserved** - never touched by the upgrade | | systemd unit file | **Replaced** - new unit file installed; `systemctl daemon-reload` run automatically | | Binary at `/usr/local/bin/dispatch` | **Replaced** - old binary overwritten atomically | @@ -3437,7 +3430,7 @@ This validation applies to `relay:{id}:smtp.host` only. Provider SDK endpoints ( - The script's SHA-256 is published on the GitHub release page and verified by a second download before execution (the install page includes a `curl | sha256sum` verification step) - The script is pinned to a specific release tag URL, not `latest`, so it cannot be silently replaced -**SQL Server Express download:** the bootstrap verifies the SHA-256 of the downloaded installer against a hardcoded value before executing it. A mismatch aborts installation with a clear error. +**PostgreSQL installer:** the bootstrap verifies the SHA-256 of the bundled (or downloaded) PostgreSQL installer against a hardcoded value before executing it. A mismatch aborts installation with a clear error. **NuGet packages:** all dependencies are locked via `packages.lock.json`. CI fails on lock file changes that are not accompanied by a manual review comment. @@ -3776,7 +3769,7 @@ app.MapFallbackToFile("index.html", new StaticFileOptions ### 19.5 Configuration Encryption -`SecureConfig` encrypts/decrypts sensitive values stored in the SQL `config` table (rows where `encrypted = 1`). The connection string in `appsettings.json` is stored in plaintext - it grants access only to a least-privilege `dispatch_app` SQL login, so exposure risk is low and file-level OS permissions provide the primary protection. +`SecureConfig` encrypts/decrypts sensitive values stored in the `config` table (rows where `encrypted = TRUE`). The connection string in `appsettings.json` is stored in plaintext - it grants access only to a least-privilege `dispatch` PostgreSQL role, so exposure risk is low and file-level OS permissions provide the primary protection. ```csharp public static class SecureConfig @@ -3789,7 +3782,7 @@ public static class SecureConfig } ``` -`ConfigRepository` calls `Encrypt()` before writing any row with `encrypted = 1` and `Decrypt()` transparently after reading. The raw plaintext never touches `appsettings.json`. Never log or surface the decrypted value - only write it to the provider SDK call. +`ConfigRepository` calls `Encrypt()` before writing any row with `encrypted = TRUE` and `Decrypt()` transparently after reading. The raw plaintext never touches `appsettings.json`. Never log or surface the decrypted value - only write it to the provider SDK call. ### 19.6 In-Memory Ring Buffer Sink (Serilog) @@ -3815,7 +3808,7 @@ public class RingBufferSink : ILogEventSink ### 19.7 Routing and Provider Resolution -The `RoutingEngine` is called on every spool file dispatch. Relay configs and routing rules are loaded from SQL via the `IRelayRepository` and `IRoutingRuleRepository` - cache them with a short TTL (e.g. 5 seconds) in the repositories so rule changes propagate quickly without a SQL query per message: +The `RoutingEngine` is called on every spool file dispatch. Relay configs and routing rules are loaded from PostgreSQL via the `IRelayRepository` and `IRoutingRuleRepository` - cache them with a short TTL (e.g. 5 seconds) in the repositories so rule changes propagate quickly without a database query per message: ```csharp // SpoolWorkerPool.ProcessAsync() @@ -3836,34 +3829,32 @@ var result = await provider.SendAsync(message, ct); `Dispatch.Bootstrap.Windows` is a .NET 10 WinForms project published as a self-contained single-file EXE. It has no .NET runtime prerequisite - use `PublishSingleFile=true` with `SelfContained=true`. -**SQL Server detection:** +**PostgreSQL detection:** ```csharp public static async Task> DetectLocalInstancesAsync() { var found = new List(); - // 1. Try well-known default instance names - foreach (var candidate in new[] { @"localhost\SQLEXPRESS", "localhost" }) + // 1. Try a PostgreSQL server on the default port + foreach (var candidate in new[] { "localhost:5432" }) { - if (await TryConnectAsync($"Server={candidate};Integrated Security=true;Timeout=2")) + var parts = candidate.Split(':'); + if (await TryConnectAsync($"Host={parts[0]};Port={parts[1]};Timeout=2")) found.Add(candidate); } - // 2. Enumerate via SQL Server Browser (best-effort - Browser may be disabled) + // 2. Enumerate any registered PostgreSQL Windows services (best-effort) try { - var table = SqlDataSourceEnumerator.Instance.GetDataSources(); - foreach (DataRow row in table.Rows) + foreach (var svc in ServiceController.GetServices()) { - var server = row["ServerName"].ToString()!; - var instance = row["InstanceName"].ToString()!; - var name = string.IsNullOrEmpty(instance) ? server : $@"{server}\{instance}"; - if (!found.Contains(name, StringComparer.OrdinalIgnoreCase)) - found.Add(name); + if (svc.ServiceName.StartsWith("postgresql", StringComparison.OrdinalIgnoreCase) && + !found.Contains(svc.ServiceName, StringComparer.OrdinalIgnoreCase)) + found.Add(svc.ServiceName); } } - catch { /* Browser disabled - ignore */ } + catch { /* service enumeration unavailable - ignore */ } return found; } @@ -3874,16 +3865,17 @@ public static async Task> DetectLocalInstancesAsync() ```csharp public static async Task InitialiseDatabaseAsync(string connectionString) { - // 1. Create database if it doesn't exist - var builder = new SqlConnectionStringBuilder(connectionString) - { InitialCatalog = "master" }; + // 1. Create database if it doesn't exist (connect to the default 'postgres' db first; + // CREATE DATABASE cannot run inside a transaction and has no IF NOT EXISTS clause). + var builder = new NpgsqlConnectionStringBuilder(connectionString) + { Database = "postgres" }; - using (var conn = new SqlConnection(builder.ToString())) + using (var conn = new NpgsqlConnection(builder.ToString())) { - await conn.ExecuteAsync(""" - IF NOT EXISTS (SELECT 1 FROM sys.databases WHERE name = 'DispatchLog') - CREATE DATABASE DispatchLog; - """); + var exists = await conn.ExecuteScalarAsync( + "SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'DispatchLog')"); + if (!exists) + await conn.ExecuteAsync("CREATE DATABASE \"DispatchLog\""); } // 2. Apply embedded schema scripts in order @@ -3892,7 +3884,7 @@ public static async Task InitialiseDatabaseAsync(string connectionString) .Where(n => n.Contains(".Schema.") && n.EndsWith(".sql")) .OrderBy(n => n); - using var db = new SqlConnection(connectionString); + using var db = new NpgsqlConnection(connectionString); foreach (var name in scripts) { using var stream = assembly.GetManifestResourceStream(name)!; @@ -4124,19 +4116,16 @@ The migration runner is shared by both the Windows bootstrap (`Dispatch.Bootstra public class MigrationRunner(string connectionString) { private const string EnsureVersionTable = """ - IF NOT EXISTS ( - SELECT 1 FROM sys.tables WHERE name = 'schema_version' - ) - CREATE TABLE schema_version ( + CREATE TABLE IF NOT EXISTS schema_version ( version INT NOT NULL PRIMARY KEY, - script_name NVARCHAR(256) NOT NULL, - applied_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + script_name VARCHAR(256) NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() ); """; public async Task RunAsync(CancellationToken ct = default) { - using var conn = new SqlConnection(connectionString); + using var conn = new NpgsqlConnection(connectionString); await conn.OpenAsync(ct); // Ensure tracking table exists @@ -4174,9 +4163,9 @@ public class MigrationRunner(string connectionString) using var tx = await conn.BeginTransactionAsync(ct); try { - // Split on GO statements (SQL Server batch separator) - foreach (var batch in SplitOnGo(sql)) - await conn.ExecuteAsync(batch, transaction: tx); + // PostgreSQL has no GO batch separator - each file is executed as one script + // (Npgsql happily runs a multi-statement command in a single ExecuteAsync). + await conn.ExecuteAsync(sql, transaction: tx); await conn.ExecuteAsync( "INSERT INTO schema_version (version, script_name) VALUES (@v, @n)", @@ -4201,10 +4190,6 @@ public class MigrationRunner(string connectionString) var filename = Path.GetFileName(resourceName); return int.Parse(filename[..4]); // e.g. "0003_add_tags.sql" → 3 } - - private static IEnumerable SplitOnGo(string sql) => - Regex.Split(sql, @"^\s*GO\s*$", RegexOptions.Multiline | RegexOptions.IgnoreCase) - .Where(b => !string.IsNullOrWhiteSpace(b)); } public record MigrationResult(int AppliedCount, List Scripts); @@ -4446,12 +4431,12 @@ When success logging is **on**, the sparkline reads from `relay_log`: ```sql -- 60 one-minute buckets, last 60 minutes - hits IX_relay_log_status_date -SELECT DATEADD(MINUTE, DATEDIFF(MINUTE, 0, logged_at), 0) AS bucket, - COUNT(*) AS delivered +SELECT date_trunc('minute', logged_at) AS bucket, + COUNT(*) AS delivered FROM relay_log WHERE status = 'OK' - AND logged_at >= DATEADD(MINUTE, -60, SYSUTCDATETIME()) -GROUP BY DATEADD(MINUTE, DATEDIFF(MINUTE, 0, logged_at), 0) + AND logged_at >= now() - INTERVAL '60 minutes' +GROUP BY date_trunc('minute', logged_at) ORDER BY bucket; ``` diff --git a/installer/README.md b/installer/README.md index 96d7a4cf..7e757db5 100644 --- a/installer/README.md +++ b/installer/README.md @@ -1,8 +1,8 @@ # Installers -Per spec §12.1, the only things written to the platform config file are the **SQL connection string**, the +Per spec §12.1, the only things written to the platform config file are the **PostgreSQL connection string**, the install-time **admin-password seed**, and (optionally) the **Web UI TLS cert**. Everything else (ports, -spool, retry, retention, …) is seeded into the SQL `config` table on first run and managed from the +spool, retry, retention, …) is seeded into the PostgreSQL `config` table on first run and managed from the dashboard. On first start the admin-password seed is hashed (bcrypt) into the `config` table; if you omit it, the dashboard shows a one-time first-run setup screen instead. @@ -14,15 +14,15 @@ Three ways to install, from most to least automated: A WiX **Burn bundle** (`DispatchSetup.exe`) that chains, in order: -1. **SQL Server Express** - `sql-express/InstallSqlExpress.exe` runs `InstallSqlExpress.ps1`, which downloads - and silently installs SQL Server Express as the named instance **`DISPATCHSQL`**, creates the - **`DispatchLog`** database, and grants `NT AUTHORITY\SYSTEM` sysadmin (the service runs as LocalSystem and - connects over shared memory with Windows auth). Skipped if the instance already exists. +1. **PostgreSQL** - `postgres/InstallPostgres.exe` runs `InstallPostgres.ps1`, which downloads + and silently installs a bundled PostgreSQL server, creates the **`DispatchLog`** database and a + least-privilege **`dispatch`** role, and configures local access for the service. Skipped if a + Dispatch PostgreSQL install already exists. 2. **Dispatch MSI** (`Dispatch.wxs`) - installs the binaries + Windows service + firewall rules and writes `%ProgramData%\Dispatch\appsettings.json` pointing at the local instance (via the `WriteAppSettings` custom action). The admin password is set on first run in the dashboard. -This pattern is adapted from the FluxDeploy installer (`packaging/sql-express-launcher`, `relay-bundle`). +This pattern is adapted from the FluxDeploy installer (`packaging/postgres-launcher`, `relay-bundle`). Build (Windows, .NET SDK + WiX v6 toolset): @@ -30,49 +30,49 @@ Build (Windows, .NET SDK + WiX v6 toolset): cd installer\windows dotnet publish ..\..\src\Dispatch.Service -c Release -o publish # (build + embed the UI first) wix build Dispatch.wxs -d PublishDir=publish -ext WixToolset.Firewall.wixext -o Dispatch.msi -dotnet build sql-express\SqlExpressLauncher.csproj -c Release # -> InstallSqlExpress.exe +dotnet build postgres\PostgresLauncher.csproj -c Release # -> InstallPostgres.exe wix build bundle\Bundle.wxs -ext WixToolset.BootstrapperApplications.wixext -ext WixToolset.Util.wixext ` - -bindpath "SqlLauncher=sql-express\bin\Release\net472" -bindpath "Msi=." -o DispatchSetup.exe + -bindpath "PostgresLauncher=postgres\bin\Release\net472" -bindpath "Msi=." -o DispatchSetup.exe ``` -To use an **existing** SQL Server instead of the local Express install, install the MSI directly and pass a -connection string: `msiexec /i Dispatch.msi SQLCONN="Server=...;Database=DispatchLog;User Id=...;Password=...;TrustServerCertificate=True;Encrypt=True"`. +To use an **existing** PostgreSQL instead of the bundled install, install the MSI directly and pass a +connection string: `msiexec /i Dispatch.msi SQLCONN="Host=localhost;Port=5432;Database=DispatchLog;Username=dispatch;Password=..."`. ### 2. MSI only (`windows/Dispatch.wxs`) -Binaries + service + firewall + `appsettings.json`. Defaults `SQLCONN` to the local `DISPATCHSQL` instance; -override via the `SQLCONN` property for an existing server. +Binaries + service + firewall + `appsettings.json`. Defaults `SQLCONN` to the local bundled PostgreSQL +instance; override via the `SQLCONN` property for an existing server. ### 3. Script (`windows/install.ps1`) Elevated PowerShell scripted install (publishes, writes config, registers the service, opens firewall) for a -pre-existing SQL Server. Requires the .NET SDK + Node.js on the host. +pre-existing PostgreSQL. Requires the .NET SDK + Node.js on the host. ## Linux (`linux/`) `install.sh` publishes the service, lays out `/opt/dispatch`, `/etc/dispatch/appsettings.json` (mode 600), `/var/lib/dispatch/spool`, `/var/log/dispatch`, creates the `dispatch` service account, and installs the -`dispatch.service` systemd unit. It can either use an existing SQL Server or install one locally. +`dispatch.service` systemd unit. It can either use an existing PostgreSQL or install one locally. ```bash -# Existing SQL Server / Azure SQL: +# Existing PostgreSQL: sudo ./linux/install.sh \ - --sql-connection "Server=...;Database=DispatchLog;User Id=...;Password=...;TrustServerCertificate=True;Encrypt=True" \ + --sql-connection "Host=...;Port=5432;Database=DispatchLog;Username=dispatch;Password=..." \ --admin-password "" -# Or install SQL Server Express locally (Ubuntu/Debian or RHEL/Fedora) + a self-signed dashboard cert: -sudo ./linux/install.sh --install-sql --sa-password "" \ +# Or install PostgreSQL locally (Ubuntu/Debian or RHEL/Fedora) + a self-signed dashboard cert: +sudo ./linux/install.sh --install-postgres --db-password "" \ --admin-password "" --generate-cert ``` -`--install-sql` adds Microsoft's `mssql-server` package (Express edition = free, via `MSSQL_PID=Express`), -runs the unattended setup, and creates the `DispatchLog` database. `--generate-cert` creates a self-signed +`--install-postgres` adds the distribution's `postgresql` package, runs the unattended setup, and creates +the `DispatchLog` database and a least-privilege `dispatch` role. `--generate-cert` creates a self-signed PFX and serves the dashboard over HTTPS (spec §17.2). Prerequisites: the .NET 10 SDK and Node.js (the script builds the embedded UI and publishes); a prebuilt self-contained tarball is future work. ## Validation status The Windows artifacts (PowerShell bootstrap, WiX MSI/bundle, the net472 launcher) and the Linux -`--install-sql` path were authored on macOS and **must be validated on the target OS** - they cannot be -built or executed here. The Linux `install.sh` passes `bash -n` syntax checks. The SQL-Express bootstrap and +`--install-postgres` path were authored on macOS and **must be validated on the target OS** - they cannot be +built or executed here. The Linux `install.sh` passes `bash -n` syntax checks. The PostgreSQL bootstrap and bundle-chaining approach mirror the proven FluxDeploy installer. diff --git a/installer/linux/dispatch-update.sh b/installer/linux/dispatch-update.sh index 27753dad..59f4f53d 100755 --- a/installer/linux/dispatch-update.sh +++ b/installer/linux/dispatch-update.sh @@ -61,7 +61,7 @@ openssl dgst -sha256 -verify "$PUBKEY" -signature "$STAGED/manifest.json.sig" "$ || fail "signature verification failed" # Authoritative version = the SIGNED manifest's version, and it must be a safe token (it lands in fs paths, -# an rm -rf target, and a T-SQL backup filename). Never trust apply.request's version for this. +# an rm -rf target, and a SQL backup filename). Never trust apply.request's version for this. VER="$(jget version "$STAGED/manifest.json")" [ -n "$VER" ] || fail "signed manifest has no version" case "$VER" in @@ -89,9 +89,9 @@ fi status "Applying" "applying $VER" # 2) Best-effort DB backup before the restart (the new version auto-applies migrations on start). Never -# blocks the update: a backup failure (e.g. external SQL, no sqlcmd) is logged and skipped. +# blocks the update: a backup failure (e.g. external DB, no pg_dump) is logged and skipped. backup_db() { - command -v sqlcmd >/dev/null 2>&1 || { echo "update: sqlcmd not found, skipping DB backup"; return 0; } + command -v pg_dump >/dev/null 2>&1 || { echo "update: pg_dump not found, skipping DB backup"; return 0; } command -v python3 >/dev/null 2>&1 || return 0 # Read the connection string via python ARGV (not interpolated into the program text). local conn; conn="$(python3 - "$DATA_DIR/appsettings.json" 2>/dev/null <<'PY' @@ -103,17 +103,19 @@ except Exception: PY )" [ -n "$conn" ] || return 0 - local srv usr pwd db; - srv="$(sed -n 's/.*[Ss]erver=\([^;]*\).*/\1/p' <<<"$conn")" - usr="$(sed -n 's/.*[Uu]ser[ ]*[Ii]d=\([^;]*\).*/\1/p' <<<"$conn")" + local host port usr pwd db; + host="$(sed -n 's/.*[Hh]ost=\([^;]*\).*/\1/p' <<<"$conn")" + port="$(sed -n 's/.*[Pp]ort=\([^;]*\).*/\1/p' <<<"$conn")" + usr="$(sed -n 's/.*[Uu]sername=\([^;]*\).*/\1/p' <<<"$conn")" pwd="$(sed -n 's/.*[Pp]assword=\([^;]*\).*/\1/p' <<<"$conn")" db="$(sed -n 's/.*[Dd]atabase=\([^;]*\).*/\1/p' <<<"$conn")" - [ -n "$srv" ] && [ -n "$usr" ] && [ -n "$db" ] || { echo "update: could not parse connection, skipping DB backup"; return 0; } - # db lands in a T-SQL identifier + a filename; require a safe token (VER was validated against the manifest). + [ -n "$port" ] || port="5432" + [ -n "$host" ] && [ -n "$usr" ] && [ -n "$db" ] || { echo "update: could not parse connection, skipping DB backup"; return 0; } + # db lands in a SQL identifier + a filename; require a safe token (VER was validated against the manifest). case "$db" in *[!A-Za-z0-9._-]*) echo "update: unsafe db name '$db', skipping DB backup"; return 0;; esac mkdir -p "$DATA_DIR/backups" local bak="$DATA_DIR/backups/${db}-pre-${VER}-$(date -u +%Y%m%dT%H%M%SZ).bak" - if sqlcmd -S "$srv" -U "$usr" -P "$pwd" -C -Q "BACKUP DATABASE [$db] TO DISK = N'$bak' WITH INIT, COMPRESSION;" >/dev/null 2>&1; then + if PGPASSWORD="$pwd" pg_dump -h "$host" -p "$port" -U "$usr" -Fc "$db" -f "$bak" >/dev/null 2>&1; then echo "update: DB backed up to $bak" else echo "update: DB backup failed (continuing) - rollback is binary-level; migrations are backward-compatible" diff --git a/installer/linux/install.sh b/installer/linux/install.sh index b5f68f1c..f099b749 100755 --- a/installer/linux/install.sh +++ b/installer/linux/install.sh @@ -3,28 +3,29 @@ # Dispatch SMTP Relay - Linux installer. # # Publishes the service, lays out config/data directories, and installs a systemd unit. Per spec §12.1 the -# only things written to appsettings.json are the SQL connection string, the install-time admin-password -# seed, and (optionally) the Web UI TLS cert - everything else lives in the SQL config table and is managed +# only things written to appsettings.json are the database connection string, the install-time admin-password +# seed, and (optionally) the Web UI TLS cert - everything else lives in the config table and is managed # from the dashboard after first run (default ports: SMTP 25 & 587, dashboard 8420, API 8025). The service # runs as the unprivileged 'dispatch' user but the systemd unit grants CAP_NET_BIND_SERVICE so it can bind # 25/587; if 25 is already taken the listener falls back to 2525 automatically. # Recommendation: install on a host with no other SMTP software (Postfix, Sendmail, Exim, …) so 25/587 are free. # # Usage: -# # Use an existing SQL Server / Azure SQL: -# sudo ./install.sh --sql-connection "Server=...;Database=DispatchLog;User Id=...;Password=...;TrustServerCertificate=True;Encrypt=True" --admin-password "" +# # Use an existing PostgreSQL server: +# sudo ./install.sh --sql-connection "Host=...;Port=5432;Database=DispatchLog;Username=...;Password=..." --admin-password "" # -# # Or have the installer set up SQL Server Express locally (Ubuntu/Debian or RHEL/Fedora): -# sudo ./install.sh --install-sql --sa-password "" --admin-password "" [--generate-cert] +# # Or have the installer set up PostgreSQL locally (Ubuntu/Debian or RHEL/Fedora): +# sudo ./install.sh --install-postgres --db-password "" --admin-password "" [--generate-cert] # # # From a release tarball (self-contained binaries; no .NET SDK / Node required on the box): -# sudo ./install.sh --prebuilt ./bin --install-sql --sa-password "" --admin-password "" +# sudo ./install.sh --prebuilt ./bin --install-postgres --db-password "" --admin-password "" # # Flags: -# --sql-connection Connection string for an existing server (omit when using --install-sql). -# --install-sql Install SQL Server (Express, free) locally + create the DispatchLog DB. On arm64 -# (no SQL Server build) this runs Azure SQL Edge in a container instead - needs Docker. -# --sa-password SA password for --install-sql (required with --install-sql; must meet SQL policy). +# --sql-connection Connection string for an existing server (omit when using --install-postgres). +# --install-postgres Install PostgreSQL locally + create the 'dispatch' role and DispatchLog DB. +# (multi-arch: works on both amd64 and arm64). Alias: --install-sql. +# --db-password Password for the created 'dispatch' role (with --install-postgres; a strong +# random one is generated if omitted). # --admin-password Dashboard admin password seed (prompted if omitted). # --generate-cert Generate a self-signed PFX and serve the dashboard over HTTPS (spec §17.2). # --http-port Firewall/URL dashboard port (default 8420; change in the dashboard to differ). @@ -34,7 +35,7 @@ # --prebuilt

Install pre-published self-contained binaries from instead of building from # source. Used by the release tarball; needs neither the .NET SDK nor Node. # --no-start Stage the install but don't start the service (the unit is enabled, not started). -# Used when baking the appliance image; first boot configures SQL and starts it. +# Used when baking the appliance image; first boot configures PostgreSQL and starts it. # set -euo pipefail @@ -51,8 +52,8 @@ SQL_CONNECTION="" ADMIN_PASSWORD="" SOURCE_DIR="" PREBUILT_DIR="" -INSTALL_SQL="0" -SA_PASSWORD="" +INSTALL_POSTGRES="0" +DB_PASSWORD="" GENERATE_CERT="0" TLS_CERT_PATH="" TLS_CERT_PASSWORD="" @@ -62,8 +63,9 @@ while [[ $# -gt 0 ]]; do case "$1" in --sql-connection) SQL_CONNECTION="$2"; shift 2;; --admin-password) ADMIN_PASSWORD="$2"; shift 2;; - --install-sql) INSTALL_SQL="1"; shift;; - --sa-password) SA_PASSWORD="$2"; shift 2;; + # --install-sql is a hidden backward-compatible alias for --install-postgres. + --install-postgres|--install-sql) INSTALL_POSTGRES="1"; shift;; + --db-password) DB_PASSWORD="$2"; shift 2;; --generate-cert) GENERATE_CERT="1"; shift;; --http-port) HTTP_PORT="$2"; shift 2;; --api-port) API_PORT="$2"; shift 2;; @@ -71,7 +73,7 @@ while [[ $# -gt 0 ]]; do --source) SOURCE_DIR="$2"; shift 2;; --prebuilt) PREBUILT_DIR="$2"; shift 2;; # Stage the install without starting the service (the unit is enabled, not started). Used when building - # the prebuilt appliance image, where SQL is configured and the service is started on first boot. + # the prebuilt appliance image, where PostgreSQL is configured and the service is started on first boot. --no-start) NO_START="1"; shift;; *) echo "Unknown option: $1" >&2; exit 1;; esac @@ -79,103 +81,60 @@ done [[ $EUID -eq 0 ]] || { echo "Please run as root (sudo)." >&2; exit 1; } -# ---- Optional: install SQL Server Express locally ----------------------------------------------- -# Installs Microsoft's mssql-server (Express edition = free, set via MSSQL_PID), runs the unattended -# setup, and creates the DispatchLog database. Best-effort across Debian/Ubuntu (apt) and RHEL/Fedora -# (dnf/yum); validate on your target distro. Sets SQL_CONNECTION to the local SA connection. -install_sql_server() { - [[ -n "$SA_PASSWORD" ]] || { echo "--sa-password is required with --install-sql." >&2; exit 1; } +# ---- Optional: install PostgreSQL locally ------------------------------------------------------- +# Installs the distro's PostgreSQL packages, ensures the cluster is running, then creates a 'dispatch' +# login role and a DispatchLog database owned by it. Best-effort across Debian/Ubuntu (apt) and +# RHEL/Fedora (dnf/yum); validate on your target distro. PostgreSQL is multi-arch (amd64 + arm64), so no +# architecture-specific fallback is needed. Sets SQL_CONNECTION to the local Npgsql connection. +install_postgres() { + # If no password was supplied, generate a strong random one for the 'dispatch' role. + if [[ -z "$DB_PASSWORD" ]]; then + DB_PASSWORD="$(openssl rand -base64 24 2>/dev/null || head -c 18 /dev/urandom | base64)" + echo "==> No --db-password given; generated a random password for the 'dispatch' role." + fi . /etc/os-release - # SQL Server has no arm64 Linux build, so on arm64 fall back to Azure SQL Edge, which is arm64-native - # but ships only as a container image. This is a dev/test convenience - SQL Edge is deprecated by - # Microsoft; for production use an amd64 host with SQL Server, or point --sql-connection at an external - # instance. local arch; arch="$(uname -m)" - if [[ "$arch" == "aarch64" || "$arch" == "arm64" ]]; then - install_sql_edge_container - return - fi - - echo "==> Installing SQL Server (Express) for $ID $VERSION_ID ($arch)" + echo "==> Installing PostgreSQL for $ID $VERSION_ID ($arch)" if command -v apt-get >/dev/null 2>&1; then - # Prerequisites the rest of this function needs (a minimal cloud/VM image often lacks gnupg + the - # https apt transport); install them before using gpg so the key import can't fail with "gpg: not found". apt-get update -y || true - apt-get install -y curl ca-certificates gnupg apt-transport-https - # SQL Server 2025 (supports Ubuntu 24.04). The 2025 .list uses signed-by=/usr/share/keyrings/microsoft-prod.gpg, - # so install the Microsoft key there. - install -d -m 0755 /usr/share/keyrings - curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --batch --yes --dearmor -o /usr/share/keyrings/microsoft-prod.gpg - chmod a+r /usr/share/keyrings/microsoft-prod.gpg - # Use the repo list for this distro release; fall back to the newest known LTS list if MS has no list - # for this exact ${VERSION_ID} yet (e.g. a brand-new Ubuntu before the SQL repo catches up). - curl -fsSL "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/mssql-server-2025.list" -o /etc/apt/sources.list.d/mssql-server-2025.list || \ - curl -fsSL "https://packages.microsoft.com/config/ubuntu/24.04/mssql-server-2025.list" -o /etc/apt/sources.list.d/mssql-server-2025.list - apt-get update -y - ACCEPT_EULA=Y MSSQL_PID=Express MSSQL_SA_PASSWORD="$SA_PASSWORD" apt-get install -y mssql-server - apt-get install -y mssql-tools18 unixodbc-dev || true + apt-get install -y postgresql postgresql-contrib elif command -v dnf >/dev/null 2>&1 || command -v yum >/dev/null 2>&1; then local PM; PM="$(command -v dnf || command -v yum)" - curl -fsSL "https://packages.microsoft.com/config/rhel/${VERSION_ID%%.*}/mssql-server-2025.repo" -o /etc/yum.repos.d/mssql-server-2025.repo - ACCEPT_EULA=Y MSSQL_PID=Express MSSQL_SA_PASSWORD="$SA_PASSWORD" "$PM" install -y mssql-server - "$PM" install -y mssql-tools18 unixODBC-devel || true + "$PM" install -y postgresql-server postgresql-contrib + # RHEL/Fedora ship an uninitialized cluster; initdb it once before starting (idempotent guard). + if [[ ! -s /var/lib/pgsql/data/PG_VERSION ]]; then + /usr/bin/postgresql-setup --initdb >/dev/null 2>&1 || postgresql-setup initdb >/dev/null 2>&1 || true + fi else - echo "Unsupported package manager - install SQL Server manually and pass --sql-connection." >&2; exit 1 + echo "Unsupported package manager - install PostgreSQL manually and pass --sql-connection." >&2; exit 1 fi - echo "==> Running unattended SQL Server setup (Express edition)" - MSSQL_PID=Express MSSQL_SA_PASSWORD="$SA_PASSWORD" ACCEPT_EULA=Y /opt/mssql/bin/mssql-conf -n setup accept-eula - systemctl enable --now mssql-server + echo "==> Ensuring the PostgreSQL cluster is started" + systemctl enable --now postgresql - echo "==> Waiting for SQL Server and creating the DispatchLog database" - local sqlcmd; sqlcmd="$(command -v sqlcmd || echo /opt/mssql-tools18/bin/sqlcmd)" + echo "==> Waiting for PostgreSQL and creating the 'dispatch' role + DispatchLog database" for _ in $(seq 1 30); do - if "$sqlcmd" -S localhost -U sa -P "$SA_PASSWORD" -C -Q "SELECT 1" >/dev/null 2>&1; then break; fi - sleep 5 - done - "$sqlcmd" -S localhost -U sa -P "$SA_PASSWORD" -C -Q "IF DB_ID('DispatchLog') IS NULL CREATE DATABASE [DispatchLog];" - SQL_CONNECTION="Server=localhost;Database=DispatchLog;User Id=sa;Password=${SA_PASSWORD};TrustServerCertificate=True;Encrypt=True" -} - -# ---- arm64 fallback: Azure SQL Edge in a container --------------------------------------------- -# SQL Server is amd64-only on Linux; Azure SQL Edge speaks the same wire protocol and runs natively on -# arm64. Needs a container runtime (docker/podman); the DispatchLog database is created by the service's -# own DatabaseInitializer on first start, so no sqlcmd is required. -install_sql_edge_container() { - echo "==> arm64 detected - SQL Server has no arm64 Linux build; using Azure SQL Edge (container) instead." - local runtime="" - if command -v docker >/dev/null 2>&1; then runtime="docker" - elif command -v podman >/dev/null 2>&1; then runtime="podman" - elif command -v apt-get >/dev/null 2>&1; then - echo "==> Installing a container runtime (docker.io)" - apt-get update -y || true - if apt-get install -y docker.io; then - systemctl enable --now docker || true - runtime="docker" - fi - fi - [[ -n "$runtime" ]] || { - echo "No container runtime found and Docker could not be installed automatically." >&2 - echo "Install Docker or Podman and re-run, or pass --sql-connection to an external SQL instance." >&2 - exit 1 - } - - echo "==> Starting Azure SQL Edge via $runtime (instance 'dispatch-sql' on port 1433)" - "$runtime" rm -f dispatch-sql >/dev/null 2>&1 || true - "$runtime" run -d --name dispatch-sql \ - -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=${SA_PASSWORD}" \ - -p 1433:1433 -v dispatch-sql-data:/var/opt/mssql \ - --restart unless-stopped \ - mcr.microsoft.com/azure-sql-edge:latest - - echo "==> Waiting for Azure SQL Edge to accept TCP on 1433" - for _ in $(seq 1 36); do - if (exec 3<>/dev/tcp/127.0.0.1/1433) 2>/dev/null; then exec 3>&- 3<&-; break; fi - sleep 5 + if sudo -u postgres psql -tAc "SELECT 1" >/dev/null 2>&1; then break; fi + sleep 2 done - # The service's DatabaseInitializer connects to master and creates DispatchLog on first start. - SQL_CONNECTION="Server=localhost,1433;Database=DispatchLog;User Id=sa;Password=${SA_PASSWORD};TrustServerCertificate=True;Encrypt=True" + # Idempotent role creation, guarded by a pg_roles check: create the login role only when absent, otherwise + # refresh its password so re-runs stay in sync with --db-password. The password is passed as a psql + # variable and quoted through format(%L), so it is safe even if it contains punctuation. \gexec runs the + # generated statement (CREATE/ALTER ROLE cannot be templated inline). + sudo -u postgres psql -v ON_ERROR_STOP=1 -v pw="$DB_PASSWORD" <<'PSQL' +SELECT format('CREATE ROLE dispatch LOGIN PASSWORD %L', :'pw') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'dispatch')\gexec +SELECT format('ALTER ROLE dispatch LOGIN PASSWORD %L', :'pw') +WHERE EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'dispatch')\gexec +PSQL + # Idempotent database creation, guarded by a pg_database check: CREATE DATABASE cannot run inside a + # transaction/DO block, so gate it with a \gexec that only fires when the database is absent. + sudo -u postgres psql -v ON_ERROR_STOP=1 <<'PSQL' +SELECT 'CREATE DATABASE "DispatchLog" OWNER dispatch' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'DispatchLog')\gexec +PSQL + SQL_CONNECTION="Host=localhost;Port=5432;Database=DispatchLog;Username=dispatch;Password=${DB_PASSWORD}" } # ---- Optional: generate a self-signed TLS cert for the dashboard -------------------------------- @@ -193,8 +152,8 @@ generate_cert() { chmod 600 "$TLS_CERT_PATH" } -[[ "$INSTALL_SQL" == "1" ]] && install_sql_server -[[ -n "$SQL_CONNECTION" ]] || { echo "Provide --sql-connection, or use --install-sql." >&2; exit 1; } +[[ "$INSTALL_POSTGRES" == "1" ]] && install_postgres +[[ -n "$SQL_CONNECTION" ]] || { echo "Provide --sql-connection, or use --install-postgres." >&2; exit 1; } # Prompt for the admin password only when omitted AND running interactively. In a non-interactive run # (e.g. baking the appliance image) leave it unset - the dashboard requires the admin password to be set on @@ -283,7 +242,7 @@ mkdir -p "$CONFIG_DIR" "$DATA_DIR/.dispatch-spool" "$LOG_DIR" echo "==> Writing $CONFIG_DIR/appsettings.json" # Spec §12.1: appsettings holds ONLY the connection string, the admin-password seed, and the Web UI TLS -# cert. Ports/spool/retry/etc. are seeded into the SQL config table on first run and managed in the dashboard. +# cert. Ports/spool/retry/etc. are seeded into the config table on first run and managed in the dashboard. WEBUI_TLS="" if [[ -n "$TLS_CERT_PATH" ]]; then WEBUI_TLS=", @@ -297,7 +256,7 @@ cat > "$CONFIG_DIR/appsettings.json" < @@ -28,10 +28,11 @@ - + + Value="Host=127.0.0.1;Port=5432;Database=DispatchLog;Username=dispatch" /> @@ -59,8 +60,9 @@ - + = 2) { var targetPath = fso.BuildPath(dataDir, "appsettings.json"); // Don't clobber an existing config (preserves manual edits + upgrades). if (!fso.FileExists(targetPath)) { - // JSON-escape the connection string (backslashes for the .\INSTANCE form, quotes). + // JSON-escape the connection string (backslashes and quotes) so any value is written safely. var conn = sqlConn.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); var json = "{\r\n" + ' "ConnectionStrings": {\r\n' diff --git a/installer/windows/bundle/Bundle.wxs b/installer/windows/bundle/Bundle.wxs index 9278c9cc..3c6e1409 100644 --- a/installer/windows/bundle/Bundle.wxs +++ b/installer/windows/bundle/Bundle.wxs @@ -1,17 +1,17 @@ - - + - - - + - + DetectCondition="PostgresInstalled"> + - diff --git a/installer/windows/functional-smoke.ps1 b/installer/windows/functional-smoke.ps1 index 3ca7ac9e..483b7afa 100644 --- a/installer/windows/functional-smoke.ps1 +++ b/installer/windows/functional-smoke.ps1 @@ -383,8 +383,8 @@ $metrics = Invoke-WebRequest -SkipCertificateCheck -WebSession $sess -Uri "$Dash if ([int]$metrics.StatusCode -ne 200) { throw "/metrics did not return 200 (got $($metrics.StatusCode))" } Write-Host "OK: stats / system / spool / health / metrics all answer" -# === 16. Size-pressure ARCHIVES then deletes (Express only) - DESTRUCTIVE, runs last ============ -# Force size-pressure by dropping the trigger below current usage, then confirm the oldest rows are +# === 16. Size-pressure ARCHIVES then deletes (opt-in DB-size cap) - DESTRUCTIVE, runs last ======== +# Force size-pressure by setting the (normally-off) trigger below current usage, then confirm the oldest rows are # exported to weekly JSONL *before* being deleted (the emergency purge never silently loses history). # This wipes the message-log/audit on the CI box, so it is intentionally the final step. if ($spoolDir -and (Test-Path -LiteralPath $spoolDir)) { @@ -398,7 +398,7 @@ if ($spoolDir -and (Test-Path -LiteralPath $spoolDir)) { if ((Get-Content -LiteralPath $archives[0].FullName -TotalCount 1) -notmatch '"event"') { throw "archived JSONL row should carry the row fields" } $rowsAfter = [int](DGet '/api/messages?event=Delivered&pageSize=1').total if ($rowsAfter -ge $rowsBefore) { throw "size-pressure should have deleted message-log rows ($rowsBefore -> $rowsAfter)" } - DPut '/api/settings' '{"retention":{"sizeTriggerGb":9.5,"sizeTargetGb":9.0}}' | Out-Null # restore + DPut '/api/settings' '{"retention":{"sizeTriggerGb":0,"sizeTargetGb":0}}' | Out-Null # restore (0 = disabled, the default) Write-Host "OK: size-pressure archived $($archives.Count) JSONL file(s) then deleted rows ($rowsBefore -> $rowsAfter)" } else { Write-Host "SKIP: spool dir not locally accessible - size-pressure archive test not exercised" } diff --git a/installer/windows/install.ps1 b/installer/windows/install.ps1 index 90bd8a77..d99689f1 100644 --- a/installer/windows/install.ps1 +++ b/installer/windows/install.ps1 @@ -4,11 +4,11 @@ .DESCRIPTION Publishes the service, writes config to %ProgramData%\Dispatch, registers a Windows Service, and - opens the firewall. The SQL connection string and the dashboard admin password are supplied at + opens the firewall. The PostgreSQL connection string and the dashboard admin password are supplied at install time (the admin password is required - you'll be prompted if it isn't passed). Run from an elevated PowerShell: - .\install.ps1 -SqlConnection "Server=...;Database=DispatchLog;User Id=sa;Password=...;TrustServerCertificate=True;Encrypt=True" + .\install.ps1 -SqlConnection "Host=localhost;Port=5432;Database=DispatchLog;Username=dispatch;Password=..." NOTE: This script has not been executed on the build machine (macOS). It is the Windows install path and should be validated on Windows. A WiX MSI (Dispatch.wxs) is also provided for packaged installs. diff --git a/installer/windows/postgres/InstallPostgres.ps1 b/installer/windows/postgres/InstallPostgres.ps1 new file mode 100644 index 00000000..788fa2d9 --- /dev/null +++ b/installer/windows/postgres/InstallPostgres.ps1 @@ -0,0 +1,119 @@ +$ErrorActionPreference = 'Stop' +# Dispatch SMTP Relay - PostgreSQL bootstrap. +# Downloads + silently installs PostgreSQL as a dedicated Windows service, then creates the DispatchLog +# database and a least-privilege 'dispatch' role that owns it. Local (loopback) connections for that role +# are trusted in pg_hba.conf, so the Dispatch service connects with no password in appsettings.json - +# mirroring the old password-less Windows-auth model, while the server only listens on localhost. +# Idempotent: skips install if the service already exists. Invoked by the WiX bundle (via +# InstallPostgres.exe) before the Dispatch MSI, or run standalone. +$logFile = 'C:\Windows\Temp\Dispatch-PostgresInstall.log' +$dbName = if ($args[0]) { $args[0] } else { 'DispatchLog' } +$role = 'dispatch' +$serviceName = 'DispatchPostgres' +$port = 5432 +$installDir = 'C:\Program Files\Dispatch\PostgreSQL' +$dataDir = 'C:\ProgramData\Dispatch\pgdata' + +# Pinned PostgreSQL 17 Windows installer (EDB). Verify the SHA-256 before running, so a tampered or +# truncated download can never be executed. Update both when bumping the pinned version. +$installerUrl = 'https://get.enterprisedb.com/postgresql/postgresql-17.2-1-windows-x64.exe' +$installerSha256 = 'REPLACE_WITH_PINNED_SHA256' # set at release time; placeholder skips the check with a warning + +function Log($msg) { "$(Get-Date -Format o) $msg" | Out-File -Append -FilePath $logFile -Encoding utf8 } + +Log '=== Dispatch PostgreSQL install started ===' +Log "Service: $serviceName Database: $dbName Role: $role Port: $port" + +if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) { + Log 'PostgreSQL service already installed' +} else { + $tempDir = 'C:\Windows\Temp' + $installer = Join-Path $tempDir 'DispatchPostgresSetup.exe' + + Log 'Downloading PostgreSQL installer...' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $installerUrl -OutFile $installer -UseBasicParsing + Log "Installer: $([math]::Round((Get-Item $installer).Length / 1MB, 1)) MB" + + if ($installerSha256 -and $installerSha256 -ne 'REPLACE_WITH_PINNED_SHA256') { + $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash + if ($actual -ne $installerSha256.ToUpper()) { + Log "ERROR: installer SHA-256 mismatch (expected $installerSha256, got $actual)" + exit 1 + } + Log 'Installer SHA-256 verified' + } else { + Log 'WARNING: installer SHA-256 not pinned - skipping integrity check' + } + + # The superuser password is generated, used only to provision the role/db below, and never persisted in + # Dispatch config (the service authenticates as the 'dispatch' role via local trust). + Add-Type -AssemblyName System.Web + $superPw = [System.Web.Security.Membership]::GeneratePassword(24, 4) + + Log 'Installing PostgreSQL (server + command-line tools only)...' + $installArgs = @( + '--mode', 'unattended', + '--unattendedmodeui', 'none', + '--prefix', "`"$installDir`"", + '--datadir', "`"$dataDir`"", + '--servicename', $serviceName, + '--serverport', "$port", + '--superpassword', "`"$superPw`"", + '--enable-components', 'server,commandlinetools', + '--disable-components', 'pgAdmin,stackbuilder' + ) + $installProc = Start-Process -FilePath $installer -ArgumentList $installArgs -Wait -PassThru -WindowStyle Hidden + Log "Installer exit code: $($installProc.ExitCode)" + if ($installProc.ExitCode -ne 0) { + Log "ERROR: PostgreSQL install failed ($($installProc.ExitCode))" + exit $installProc.ExitCode + } + + Remove-Item $installer -Force -EA SilentlyContinue + + $psql = Join-Path $installDir 'bin\psql.exe' + $env:PGPASSWORD = $superPw + + Log 'Waiting for PostgreSQL to accept connections...' + $ready = $false + for ($i = 0; $i -lt 60; $i++) { + & (Join-Path $installDir 'bin\pg_isready.exe') -h 127.0.0.1 -p $port -U postgres 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { $ready = $true; break } + Start-Sleep -Seconds 2 + } + if (-not $ready) { Log 'ERROR: PostgreSQL did not become ready'; exit 1 } + + Log "Creating role [$role] and database [$dbName]..." + # Role: idempotent via a DO block (a single statement, so it is safe with psql -c). CREATE DATABASE + # cannot run inside DO/a transaction, so guard it with a separate existence check + create. + $roleSql = "DO `$`$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '$role') THEN CREATE ROLE $role LOGIN; END IF; END `$`$;" + & $psql -h 127.0.0.1 -p $port -U postgres -d postgres -v ON_ERROR_STOP=1 -c $roleSql 2>&1 | ForEach-Object { Log $_ } + + $dbExists = (& $psql -h 127.0.0.1 -p $port -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$dbName'") | Out-String + if ($dbExists.Trim() -ne '1') { + & $psql -h 127.0.0.1 -p $port -U postgres -d postgres -v ON_ERROR_STOP=1 -c "CREATE DATABASE ""$dbName"" OWNER $role;" 2>&1 | ForEach-Object { Log $_ } + } + Remove-Item Env:\PGPASSWORD -EA SilentlyContinue + + # Restrict the server to loopback and trust the dispatch role locally (no password in Dispatch config). + Log 'Configuring pg_hba.conf (localhost trust for the dispatch role) + listen_addresses...' + $hba = Join-Path $dataDir 'pg_hba.conf' + $conf = Join-Path $dataDir 'postgresql.conf' + @( + "# Dispatch: trust the local service role over loopback only.", + "host $dbName $role 127.0.0.1/32 trust", + "host $dbName $role ::1/128 trust" + ) | Add-Content -Path $hba -Encoding ascii + Add-Content -Path $conf -Value "listen_addresses = 'localhost'" -Encoding ascii + Restart-Service -Name $serviceName -Force +} + +Log 'Waiting for PostgreSQL service to be running...' +for ($i = 0; $i -lt 60; $i++) { + if (Get-Service -Name $serviceName -EA SilentlyContinue | Where-Object { $_.Status -eq 'Running' }) { Log 'PostgreSQL running'; break } + Start-Sleep -Seconds 2 +} + +Log '=== Dispatch PostgreSQL install complete ===' +exit 0 diff --git a/installer/windows/sql-express/SqlExpressLauncher.cs b/installer/windows/postgres/PostgresLauncher.cs similarity index 73% rename from installer/windows/sql-express/SqlExpressLauncher.cs rename to installer/windows/postgres/PostgresLauncher.cs index 618afd0d..b75f25cf 100644 --- a/installer/windows/sql-express/SqlExpressLauncher.cs +++ b/installer/windows/postgres/PostgresLauncher.cs @@ -2,14 +2,14 @@ using System.Diagnostics; using System.IO; -// Thin launcher (adapted from FluxDeploy): a WiX Burn ExePackage runs an EXE, not a .ps1, so this -// wrapper invokes InstallSqlExpress.ps1 (shipped alongside it as a bundle payload). Arg 0 = DB name. +// Thin launcher: a WiX Burn ExePackage runs an EXE, not a .ps1, so this wrapper invokes +// InstallPostgres.ps1 (shipped alongside it as a bundle payload). Arg 0 = DB name. class Program { static int Main(string[] args) { var scriptDir = AppDomain.CurrentDomain.BaseDirectory; - var scriptPath = Path.Combine(scriptDir, "InstallSqlExpress.ps1"); + var scriptPath = Path.Combine(scriptDir, "InstallPostgres.ps1"); if (!File.Exists(scriptPath)) { Console.Error.WriteLine($"Script not found: {scriptPath}"); diff --git a/installer/windows/sql-express/SqlExpressLauncher.csproj b/installer/windows/postgres/PostgresLauncher.csproj similarity index 74% rename from installer/windows/sql-express/SqlExpressLauncher.csproj rename to installer/windows/postgres/PostgresLauncher.csproj index 0249c362..af568f00 100644 --- a/installer/windows/sql-express/SqlExpressLauncher.csproj +++ b/installer/windows/postgres/PostgresLauncher.csproj @@ -2,13 +2,13 @@ Exe net472 - InstallSqlExpress + InstallPostgres x64 - + diff --git a/installer/windows/sql-express/InstallSqlExpress.ps1 b/installer/windows/sql-express/InstallSqlExpress.ps1 deleted file mode 100644 index 45c4e300..00000000 --- a/installer/windows/sql-express/InstallSqlExpress.ps1 +++ /dev/null @@ -1,109 +0,0 @@ -$ErrorActionPreference = 'Stop' -# Dispatch SMTP Relay - SQL Server Express bootstrap (adapted from the FluxDeploy installer pattern). -# Downloads + silently installs SQL Server Express as a named instance, then creates the DispatchLog -# database and grants the service account (LocalSystem) sysadmin so the service can connect with -# Windows auth. Idempotent: skips install if the instance already exists. Invoked by the WiX bundle -# (via InstallSqlExpress.exe) before the Dispatch MSI, or run standalone. -$logFile = 'C:\Windows\Temp\Dispatch-SqlInstall.log' -$dbName = if ($args[0]) { $args[0] } else { 'DispatchLog' } -$instance = 'DISPATCHSQL' -$serviceName = "MSSQL`$$instance" - -function Log($msg) { "$(Get-Date -Format o) $msg" | Out-File -Append -FilePath $logFile -Encoding utf8 } - -Log '=== Dispatch SQL Server Express install started ===' -Log "Instance: $instance Database: $dbName User: $([Environment]::UserName)" - -if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) { - Log 'SQL Server Express instance already installed' -} else { - $tempDir = 'C:\Windows\Temp' - $bootstrapper = Join-Path $tempDir 'SQLEXPR-SSEI.exe' - $mediaDir = Join-Path $tempDir 'DispatchSqlMedia' - $extractDir = Join-Path $tempDir 'DispatchSqlSetup' - - Log 'Downloading SQL Server Express bootstrapper...' - # fwlink 2216019 resolves to aka.ms/sql2025express (the current SQL Server 2025 Express SSEI bootstrapper). - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - Invoke-WebRequest -Uri 'https://go.microsoft.com/fwlink/?linkid=2216019' -OutFile $bootstrapper -UseBasicParsing - Log "Bootstrapper: $([math]::Round((Get-Item $bootstrapper).Length / 1MB, 1)) MB" - - Log 'Downloading SQL Server Express media...' - New-Item -Path $mediaDir -ItemType Directory -Force | Out-Null - $dlJob = Start-Process -FilePath $bootstrapper -ArgumentList '/ACTION=Download',"/MEDIAPATH=$mediaDir",'/MEDIATYPE=Core','/QUIET' -Wait -PassThru -WindowStyle Hidden - Log "Bootstrapper exit: $($dlJob.ExitCode)" - $mediaExe = Get-ChildItem $mediaDir -Filter 'SQLEXPR*.exe' -EA SilentlyContinue | Select-Object -First 1 - if (-not $mediaExe) { Log 'ERROR: media download failed'; exit 1 } - Log "Media: $($mediaExe.Name) ($([math]::Round($mediaExe.Length / 1MB)) MB)" - - Log 'Extracting...' - New-Item -Path $extractDir -ItemType Directory -Force | Out-Null - $p = Start-Process -FilePath $mediaExe.FullName -ArgumentList '/q',"/x:$extractDir" -Wait -PassThru -WindowStyle Hidden - Log "Extract exit: $($p.ExitCode)" - $setupExe = Get-ChildItem $extractDir -Filter 'SETUP.EXE' -Recurse -EA SilentlyContinue | Select-Object -First 1 - if (-not $setupExe) { Log 'ERROR: SETUP.EXE not found'; exit 1 } - - # SQL 2025 changed the extraction layout - make sure MediaInfo.xml is two levels above SETUP.EXE. - $expectedMediaInfo = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($setupExe.DirectoryName, '..', '..', 'MediaInfo.xml')) - if (-not (Test-Path $expectedMediaInfo)) { - $foundMediaInfo = Get-ChildItem $extractDir -Filter 'MediaInfo.xml' -Recurse -EA SilentlyContinue | Select-Object -First 1 - if ($foundMediaInfo) { Copy-Item $foundMediaInfo.FullName $expectedMediaInfo -Force; Log "Placed MediaInfo.xml at $expectedMediaInfo" } - else { Log "WARNING: MediaInfo.xml not found in $extractDir" } - } - - try { Remove-Item 'HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\170\ConfigurationState' -Recurse -Force -EA SilentlyContinue } catch {} - try { Remove-Item 'HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\160\ConfigurationState' -Recurse -Force -EA SilentlyContinue } catch {} - - Log 'Installing SQL Server Express...' - # Grant BUILTIN\ADMINISTRATORS and NT AUTHORITY\SYSTEM sysadmin so the Dispatch service (LocalSystem) - # can connect with Windows auth. TCP disabled - the service connects over shared memory locally. - $installArgs = '/Q /IACCEPTSQLSERVERLICENSETERMS /ACTION=Install /FEATURES=SQLENGINE ' + - "/INSTANCENAME=$instance " + - '/SQLSVCACCOUNT="NT AUTHORITY\SYSTEM" ' + - '/SQLSYSADMINACCOUNTS="BUILTIN\ADMINISTRATORS" "NT AUTHORITY\SYSTEM" ' + - '/TCPENABLED=0 /UPDATEENABLED=False /USEMICROSOFTUPDATE=False' - Log "Args: $installArgs" - $installProc = Start-Process -FilePath $setupExe.FullName -ArgumentList $installArgs -Wait -PassThru -WindowStyle Hidden - Log "SETUP.EXE exit code: $($installProc.ExitCode)" - if ($installProc.ExitCode -ne 0) { - Log "ERROR: SQL Server Express install failed ($($installProc.ExitCode))" - $summary = Get-ChildItem 'C:\Program Files\Microsoft SQL Server' -Filter 'Summary*.txt' -Recurse -EA SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 - if ($summary) { Get-Content $summary.FullName | Select-Object -First 20 | Out-File -Append $logFile -Encoding utf8 } - exit $installProc.ExitCode - } - - Remove-Item $bootstrapper -Force -EA SilentlyContinue - Remove-Item $mediaDir -Recurse -Force -EA SilentlyContinue - Remove-Item $extractDir -Recurse -Force -EA SilentlyContinue -} - -Log 'Waiting for SQL Server Express service...' -for ($i = 0; $i -lt 120; $i++) { - if (Get-Service -Name $serviceName -EA SilentlyContinue | Where-Object { $_.Status -eq 'Running' }) { Log 'SQL Express running'; break } - Start-Sleep -Seconds 5 -} - -Log "Creating database [$dbName] + ensuring NT AUTHORITY\SYSTEM login..." -for ($attempt = 1; $attempt -le 6; $attempt++) { - try { - $conn = New-Object System.Data.SqlClient.SqlConnection("Server=.\$instance;Database=master;Trusted_Connection=yes;") - $conn.Open() - $cmd = $conn.CreateCommand() - $cmd.CommandText = @" -IF DB_ID('$dbName') IS NULL CREATE DATABASE [$dbName]; -IF NOT EXISTS (SELECT 1 FROM sys.server_principals WHERE name = 'NT AUTHORITY\SYSTEM') - CREATE LOGIN [NT AUTHORITY\SYSTEM] FROM WINDOWS; -EXEC sp_addsrvrolemember 'NT AUTHORITY\SYSTEM', 'sysadmin'; -"@ - $cmd.ExecuteNonQuery() | Out-Null - $conn.Close() - Log "Database [$dbName] ready" - break - } catch { - Log "DB attempt $attempt/6: $($_.Exception.Message)" - Start-Sleep -Seconds 10 - } -} - -Log '=== Dispatch SQL Server Express install complete ===' -exit 0 diff --git a/src/Dispatch.Core/Configuration/ConfigDefaults.cs b/src/Dispatch.Core/Configuration/ConfigDefaults.cs index c0f029ee..d4186ca9 100644 --- a/src/Dispatch.Core/Configuration/ConfigDefaults.cs +++ b/src/Dispatch.Core/Configuration/ConfigDefaults.cs @@ -65,8 +65,8 @@ public static class ConfigDefaults [ConfigKeys.PurgeAuditRetentionDays] = "7", [ConfigKeys.PurgeAuditSecurityRetentionDays] = "7", [ConfigKeys.PurgeArchiveRetentionDays] = "0", // size-pressure JSONL archives: 0 = keep forever - [ConfigKeys.PurgeSizeTriggerGb] = "9.5", - [ConfigKeys.PurgeSizeTargetGb] = "9.0", + [ConfigKeys.PurgeSizeTriggerGb] = "0", // 0 = size-pressure disabled (Postgres has no hard size cap; opt-in) + [ConfigKeys.PurgeSizeTargetGb] = "0", [ConfigKeys.UpdatesSelfManaged] = "false", // installers flip this on where a platform updater ships }; diff --git a/src/Dispatch.Core/Configuration/PurgeOptions.cs b/src/Dispatch.Core/Configuration/PurgeOptions.cs index 7f23d960..201ed123 100644 --- a/src/Dispatch.Core/Configuration/PurgeOptions.cs +++ b/src/Dispatch.Core/Configuration/PurgeOptions.cs @@ -20,7 +20,7 @@ public sealed class PurgeOptions /// Noisier security audit events (allow-list denials, SMTP auth failures) kept this long (0 = forever). public int AuditSecurityRetentionDays { get; set; } = 7; - /// Weekly JSONL archive files written by the Express size-pressure purge are deleted after this + /// Weekly JSONL archive files written by the size-pressure purge are deleted after this /// many days (0 = keep forever - the default, since they're emergency exports). public int ArchiveRetentionDays { get; set; } @@ -37,7 +37,10 @@ public sealed class LogRetention public sealed class SizePressureOptions { - public double TriggerGb { get; set; } = 9.5; // SQL Server Express 10 GB limit − 0.5 GB buffer - public double TargetGb { get; set; } = 9.0; + // Optional physical DB-size cap (pg_database_size). 0 = disabled (the default): PostgreSQL has no + // hard size limit, so size-pressure is opt-in. Set TriggerGb > 0 to have the purge archive + delete + // the oldest history (then VACUUM FULL) once the database grows past TriggerGb, down to TargetGb. + public double TriggerGb { get; set; } + public double TargetGb { get; set; } } } diff --git a/src/Dispatch.Core/Logging/ILogMaintenance.cs b/src/Dispatch.Core/Logging/ILogMaintenance.cs index 9e8b397b..39806153 100644 --- a/src/Dispatch.Core/Logging/ILogMaintenance.cs +++ b/src/Dispatch.Core/Logging/ILogMaintenance.cs @@ -9,21 +9,22 @@ public interface ILogMaintenance /// batches with a short pause, to avoid lock contention. Returns total rows deleted. Task PurgeByRetentionAsync(string @event, int retentionDays, CancellationToken ct = default); - /// Approximate database file size in bytes (allocated; for the /health + storage display). + /// Physical database size in bytes (pg_database_size; for the /health + storage display + /// and the optional size-pressure trigger). Task GetDatabaseSizeBytesAsync(CancellationToken ct = default); - /// Approximate USED bytes inside the database files (drops as rows are deleted). The - /// size-pressure purge keys off this so it frees a bounded amount and terminates, rather than the - /// allocated file size (which SQL Server never shrinks on DELETE). - Task GetDatabaseUsedBytesAsync(CancellationToken ct = default); - - /// True only when the database is a SQL Server Express edition, which caps each database's data - /// files at 10 GB. The size-pressure purge applies only here; a Standard/Enterprise/Azure/external server - /// has no such cap, so size-pressure is skipped entirely. - Task IsSizeCappedEditionAsync(CancellationToken ct = default); + /// Physical bytes and live row count of relay_log (pg_total_relation_size + + /// count(*)), used to estimate how many oldest rows to free to hit a target size. + Task<(long TableBytes, long RowCount)> GetRelayLogStatsAsync(CancellationToken ct = default); /// Size-pressure step: reads the oldest relay_log rows, hands them to /// (which must persist them) and only then deletes them. Returns rows deleted. /// If archiving throws, nothing is deleted. Task ArchiveAndDeleteOldestRelayLogAsync(int batch, ArchiveRows archive, CancellationToken ct = default); + + /// Reclaims physical space after a size-pressure purge by running VACUUM FULL on the log + /// tables, so pg_database_size actually shrinks and the size-pressure trigger clears. Postgres does + /// not return space to the OS on plain DELETE/VACUUM; this is invoked only from the opt-in size-pressure + /// path (it briefly takes an exclusive lock on the log tables). + Task VacuumLogTablesAsync(CancellationToken ct = default); } diff --git a/src/Dispatch.Core/Maintenance/IStorageReport.cs b/src/Dispatch.Core/Maintenance/IStorageReport.cs index 693b69a8..884d2b47 100644 --- a/src/Dispatch.Core/Maintenance/IStorageReport.cs +++ b/src/Dispatch.Core/Maintenance/IStorageReport.cs @@ -5,7 +5,7 @@ public sealed record LogEventCount(string Event, long Rows); /// /// Database-side storage usage, so operators can see what their retention windows are actually holding -/// (spec §6.10 companion to retention). Per-event byte figures aren't cheap to get exactly in SQL Server, +/// (spec §6.10 companion to retention). Per-event byte figures aren't cheap to get exactly in PostgreSQL, /// so the endpoint estimates them by apportioning the relay_log table size across the event row counts; /// these raw facts are what that estimate is built from. /// diff --git a/src/Dispatch.Core/Maintenance/PurgeWorker.cs b/src/Dispatch.Core/Maintenance/PurgeWorker.cs index e5f89cd1..f7cba97d 100644 --- a/src/Dispatch.Core/Maintenance/PurgeWorker.cs +++ b/src/Dispatch.Core/Maintenance/PurgeWorker.cs @@ -9,8 +9,8 @@ namespace Dispatch.Core.Maintenance; /// /// Background retention + auto-purge (spec §6.10): deletes aged spool/failed and spool/captured files, -/// purges old relay_log rows per event type, and runs a size-pressure purge when the database nears the -/// SQL Server Express limit. Runs once on startup and then on a fixed interval. All work is best-effort - +/// purges old relay_log rows per event type, and runs an optional size-pressure purge when the database +/// exceeds a configured size cap. Runs once on startup and then on a fixed interval. All work is best-effort - /// a failure in one cycle is logged and never crashes the service. /// public sealed class PurgeWorker( @@ -108,35 +108,44 @@ private async Task PurgeLogAsync(string @event, int retentionDays, Cancella private async Task RunSizePressureAsync(PurgeOptions o, CancellationToken ct) { - // The 10 GB data-file cap only exists on SQL Server Express. On Standard/Enterprise/Azure or an - // operator's own external server there is no cap, so the size-pressure purge never runs there. - if (!await logs.IsSizeCappedEditionAsync(ct)) return 0; + // Optional physical DB-size cap. PostgreSQL has no hard size limit, so this is opt-in: a TriggerGb of + // 0 (the default) disables it entirely. When set, keep the database at roughly TargetGb by archiving + // and deleting the oldest history once pg_database_size exceeds TriggerGb. + if (o.SizePressure.TriggerGb <= 0) return 0; - // Key off USED space, not the allocated file size (which never shrinks on DELETE) - so the loop - // actually converges and frees a bounded amount. - var used = await logs.GetDatabaseUsedBytesAsync(ct); + var size = await logs.GetDatabaseSizeBytesAsync(ct); var trigger = (long)(o.SizePressure.TriggerGb * BytesPerGb); - if (used < trigger) return 0; + if (size < trigger) return 0; var targetGb = Math.Min(o.SizePressure.TargetGb, o.SizePressure.TriggerGb); var target = (long)(targetGb * BytesPerGb); - log.LogWarning("Database using {UsedGb:F1} GB (Express 10 GB cap) - archiving + purging oldest rows down to {TargetGb:F1} GB", - used / (double)BytesPerGb, targetGb); + // pg_database_size does not drop on DELETE (only on VACUUM FULL), so we cannot loop on it. Instead + // estimate how many oldest rows to free from the size overage and the average relay_log row size, + // delete that many in bounded batches, then VACUUM FULL once so the size actually shrinks. + var overage = size - target; + var (relayBytes, relayRows) = await logs.GetRelayLogStatsAsync(ct); + var avgRowBytes = relayRows > 0 ? Math.Max(1, relayBytes / relayRows) : 1; + var rowsToFree = (int)Math.Min(int.MaxValue, Math.Max(0, overage / avgRowBytes)); + if (rowsToFree == 0) return 0; + + log.LogWarning("Database at {SizeGb:F1} GB exceeds the {TriggerGb:F1} GB cap - archiving + purging ~{Rows} oldest rows down to {TargetGb:F1} GB", + size / (double)BytesPerGb, o.SizePressure.TriggerGb, rowsToFree, targetGb); // Archive rows to weekly JSONL before deleting them, so this emergency purge never silently loses // history. Message-log first (usually the bulk), then audit when relay_log is exhausted. var archive = new JsonlRowArchive(spool.ArchiveDir); int total = 0, archivedRelay = 0, archivedAudit = 0; - while (!ct.IsCancellationRequested && await logs.GetDatabaseUsedBytesAsync(ct) > target) + while (!ct.IsCancellationRequested && total < rowsToFree) { - var n = await logs.ArchiveAndDeleteOldestRelayLogAsync(500, + var batch = Math.Min(500, rowsToFree - total); + var n = await logs.ArchiveAndDeleteOldestRelayLogAsync(batch, (rows, _) => { archive.Append("relay_log", rows, "logged_at"); return Task.CompletedTask; }, ct); if (n > 0) { archivedRelay += n; total += n; await Task.Delay(100, ct); continue; } if (audit is not null) { - n = await audit.ArchiveAndDeleteOldestAsync(500, + n = await audit.ArchiveAndDeleteOldestAsync(batch, (rows, _) => { archive.Append("audit_log", rows, "logged_at"); return Task.CompletedTask; }, ct); if (n > 0) { archivedAudit += n; total += n; await Task.Delay(100, ct); continue; } } @@ -145,11 +154,15 @@ private async Task RunSizePressureAsync(PurgeOptions o, CancellationToken c if (total > 0) { + // Reclaim the freed space to the OS so pg_database_size drops below the trigger for next cycle. + try { await logs.VacuumLogTablesAsync(ct); } + catch (Exception ex) { log.LogWarning(ex, "VACUUM FULL after size-pressure purge failed"); } + log.LogWarning("Size-pressure purge archived + deleted {Relay} message-log and {Audit} audit rows to {Dir}", archivedRelay, archivedAudit, spool.ArchiveDir); if (audit is not null) await audit.Lifecycle("Size-pressure cleanup ran", - $"Database neared the 10 GB SQL Express cap; archived + deleted {archivedRelay} message-log and {archivedAudit} audit rows to {spool.ArchiveDir}.", + $"Database exceeded the configured {o.SizePressure.TriggerGb:F1} GB cap; archived + deleted {archivedRelay} message-log and {archivedAudit} audit rows to {spool.ArchiveDir}.", "Warning"); } return total; diff --git a/src/Dispatch.Data/DatabaseInitializer.cs b/src/Dispatch.Data/DatabaseInitializer.cs index 35c67132..e940565c 100644 --- a/src/Dispatch.Data/DatabaseInitializer.cs +++ b/src/Dispatch.Data/DatabaseInitializer.cs @@ -1,6 +1,5 @@ -using System.Reflection; using Dapper; -using Microsoft.Data.SqlClient; +using Npgsql; using Microsoft.Extensions.Logging; namespace Dispatch.Data; @@ -16,28 +15,30 @@ public sealed class DatabaseInitializer(SqlConnectionFactory factory, ILogger( - "SELECT 1 FROM sys.databases WHERE name = @database", new { database }); + "SELECT 1 FROM pg_database WHERE datname = @database", new { database }); if (exists is null) { - // CREATE DATABASE cannot be parameterised; database name comes from our own config, not user input. - await cn.ExecuteAsync($"CREATE DATABASE [{database.Replace("]", "]]")}]"); + // CREATE DATABASE cannot be parameterised or run inside a transaction; the database name comes + // from our own config, not user input. Double-quote to allow mixed case / reserved words. + await cn.ExecuteAsync($"CREATE DATABASE \"{database.Replace("\"", "\"\"")}\""); log.LogInformation("Created database {Database}", database); } } @@ -46,11 +47,10 @@ private async Task EnsureSchemaVersionTableAsync(CancellationToken ct) { await using var cn = await factory.OpenAsync(ct); await cn.ExecuteAsync(""" - IF OBJECT_ID('schema_version') IS NULL - CREATE TABLE schema_version ( - version INT NOT NULL PRIMARY KEY, - script_name NVARCHAR(256) NOT NULL, - applied_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + CREATE TABLE IF NOT EXISTS schema_version ( + version int NOT NULL PRIMARY KEY, + script_name varchar(256) NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() ); """); } @@ -83,8 +83,8 @@ await cn.ExecuteAsync( } } - /// Opens a connection, retrying for up to ~60s so a just-started SQL container (CI/docker) is tolerated. - private async Task OpenWithRetryAsync(SqlConnection cn, CancellationToken ct) + /// Opens a connection, retrying for up to ~60s so a just-started Postgres container (CI/docker) is tolerated. + private async Task OpenWithRetryAsync(NpgsqlConnection cn, CancellationToken ct) { const int maxAttempts = 30; for (var attempt = 1; ; attempt++) @@ -94,9 +94,9 @@ private async Task OpenWithRetryAsync(SqlConnection cn, CancellationToken ct) await cn.OpenAsync(ct); return; } - catch (SqlException) when (attempt < maxAttempts) + catch (NpgsqlException) when (attempt < maxAttempts) { - if (attempt == 1) log.LogInformation("Waiting for SQL Server to accept connections…"); + if (attempt == 1) log.LogInformation("Waiting for PostgreSQL to accept connections…"); await Task.Delay(TimeSpan.FromSeconds(2), ct); } } diff --git a/src/Dispatch.Data/Dispatch.Data.csproj b/src/Dispatch.Data/Dispatch.Data.csproj index a3d0c6ef..c2365238 100644 --- a/src/Dispatch.Data/Dispatch.Data.csproj +++ b/src/Dispatch.Data/Dispatch.Data.csproj @@ -15,10 +15,10 @@ - + diff --git a/src/Dispatch.Data/Migrations/0001_init.sql b/src/Dispatch.Data/Migrations/0001_init.sql index 8c346d3b..877767e7 100644 --- a/src/Dispatch.Data/Migrations/0001_init.sql +++ b/src/Dispatch.Data/Migrations/0001_init.sql @@ -1,93 +1,93 @@ --- Dispatch schema v1 (spec §6.11, §7.6, §12.3). Runs as a single batch (no GO separators); +-- Dispatch schema v1 (spec §6.11, §7.6, §12.3). Runs as a single batch; -- tables are created in FK-dependency order. Guarded by schema_version, so it runs once. CREATE TABLE config ( - [key] NVARCHAR(128) NOT NULL PRIMARY KEY, - value NVARCHAR(MAX) NOT NULL, - encrypted BIT NOT NULL DEFAULT 0, - updated_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + "key" varchar(128) NOT NULL PRIMARY KEY, + value text NOT NULL, + encrypted boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now() ); CREATE TABLE relays ( - id INT IDENTITY PRIMARY KEY, - name NVARCHAR(128) NOT NULL UNIQUE, - provider NVARCHAR(64) NOT NULL, - is_default BIT NOT NULL DEFAULT 0, - enabled BIT NOT NULL DEFAULT 1, - max_concurrency INT NOT NULL DEFAULT 4, - max_message_bytes INT NOT NULL DEFAULT 0, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - updated_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name varchar(128) NOT NULL UNIQUE, + provider varchar(64) NOT NULL, + is_default boolean NOT NULL DEFAULT false, + enabled boolean NOT NULL DEFAULT true, + max_concurrency int NOT NULL DEFAULT 4, + max_message_bytes int NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() ); -CREATE UNIQUE INDEX IX_relays_default ON relays (is_default) WHERE is_default = 1; +CREATE UNIQUE INDEX IX_relays_default ON relays (is_default) WHERE is_default; CREATE TABLE routing_rules ( - id INT IDENTITY PRIMARY KEY, - priority INT NOT NULL UNIQUE, - name NVARCHAR(128) NOT NULL, - recipient_pattern NVARCHAR(256) NULL, - sender_pattern NVARCHAR(256) NULL, - relay_id INT NOT NULL REFERENCES relays(id), - enabled BIT NOT NULL DEFAULT 1, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME() + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + priority int NOT NULL UNIQUE, + name varchar(128) NOT NULL, + recipient_pattern varchar(256) NULL, + sender_pattern varchar(256) NULL, + relay_id int NOT NULL REFERENCES relays(id), + enabled boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now() ); CREATE TABLE api_keys ( - id INT IDENTITY PRIMARY KEY, - key_id NVARCHAR(32) NOT NULL UNIQUE, - key_hash NVARCHAR(512) NOT NULL, - name NVARCHAR(256) NOT NULL, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - last_used_at DATETIME2 NULL, - message_count BIGINT NOT NULL DEFAULT 0, - revoked BIT NOT NULL DEFAULT 0, - revoked_at DATETIME2 NULL, - rate_limit_per_minute INT NOT NULL DEFAULT 100, - scope NVARCHAR(64) NOT NULL DEFAULT 'send' + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + key_id varchar(32) NOT NULL UNIQUE, + key_hash varchar(512) NOT NULL, + name varchar(256) NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + last_used_at timestamptz NULL, + message_count bigint NOT NULL DEFAULT 0, + revoked boolean NOT NULL DEFAULT false, + revoked_at timestamptz NULL, + rate_limit_per_minute int NOT NULL DEFAULT 100, + scope varchar(64) NOT NULL DEFAULT 'send' ); -CREATE INDEX IX_api_keys_key_id ON api_keys (key_id) WHERE revoked = 0; +CREATE INDEX IX_api_keys_key_id ON api_keys (key_id) WHERE NOT revoked; CREATE TABLE relay_counters ( - id INT IDENTITY PRIMARY KEY, - date DATE NOT NULL, - relay_id INT NOT NULL REFERENCES relays(id), - received BIGINT NOT NULL DEFAULT 0, - delivered BIGINT NOT NULL DEFAULT 0, - failed BIGINT NOT NULL DEFAULT 0, - retried BIGINT NOT NULL DEFAULT 0, - denied BIGINT NOT NULL DEFAULT 0, + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + date date NOT NULL, + relay_id int NOT NULL REFERENCES relays(id), + received bigint NOT NULL DEFAULT 0, + delivered bigint NOT NULL DEFAULT 0, + failed bigint NOT NULL DEFAULT 0, + retried bigint NOT NULL DEFAULT 0, + denied bigint NOT NULL DEFAULT 0, CONSTRAINT UQ_relay_counters UNIQUE (date, relay_id) ); CREATE INDEX IX_relay_counters_date ON relay_counters (date DESC); CREATE TABLE relay_log ( - id BIGINT IDENTITY PRIMARY KEY, - logged_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - spool_id NVARCHAR(64) NOT NULL, - event NVARCHAR(32) NOT NULL, - status NVARCHAR(16) NOT NULL, - retry_attempt INT NOT NULL DEFAULT 0, - from_address NVARCHAR(512) NOT NULL, - from_domain NVARCHAR(255) NOT NULL, - to_addresses NVARCHAR(MAX) NOT NULL, - to_domain NVARCHAR(255) NOT NULL, - subject NVARCHAR(998) NOT NULL, - size_bytes INT NOT NULL DEFAULT 0, - relay_id INT NULL REFERENCES relays(id), - relay_name NVARCHAR(128) NULL, - routing_rule_id INT NULL REFERENCES routing_rules(id), - routing_rule_name NVARCHAR(128) NULL, - routing_matched BIT NOT NULL DEFAULT 0, - provider NVARCHAR(64) NULL, - provider_message_id NVARCHAR(256) NULL, - provider_response NVARCHAR(MAX) NULL, - duration_ms INT NULL, - error NVARCHAR(MAX) NULL, - ingest_source NVARCHAR(16) NOT NULL DEFAULT 'SMTP', - source_ip NVARCHAR(64) NULL, - api_key_id INT NULL REFERENCES api_keys(id), - api_key_name NVARCHAR(256) NULL, - tags NVARCHAR(MAX) NULL + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + logged_at timestamptz NOT NULL DEFAULT now(), + spool_id varchar(64) NOT NULL, + event varchar(32) NOT NULL, + status varchar(16) NOT NULL, + retry_attempt int NOT NULL DEFAULT 0, + from_address varchar(512) NOT NULL, + from_domain varchar(255) NOT NULL, + to_addresses text NOT NULL, + to_domain varchar(255) NOT NULL, + subject varchar(998) NOT NULL, + size_bytes int NOT NULL DEFAULT 0, + relay_id int NULL REFERENCES relays(id), + relay_name varchar(128) NULL, + routing_rule_id int NULL REFERENCES routing_rules(id), + routing_rule_name varchar(128) NULL, + routing_matched boolean NOT NULL DEFAULT false, + provider varchar(64) NULL, + provider_message_id varchar(256) NULL, + provider_response text NULL, + duration_ms int NULL, + error text NULL, + ingest_source varchar(16) NOT NULL DEFAULT 'SMTP', + source_ip varchar(64) NULL, + api_key_id int NULL REFERENCES api_keys(id), + api_key_name varchar(256) NULL, + tags text NULL ); CREATE INDEX IX_relay_log_status_date ON relay_log (status, logged_at DESC) INCLUDE (spool_id, from_address, from_domain, to_domain, subject, size_bytes, @@ -98,14 +98,14 @@ CREATE INDEX IX_relay_log_source ON relay_log (ingest_source, logged_at DES CREATE INDEX IX_relay_log_purge ON relay_log (logged_at); CREATE TABLE config_smtp_credentials ( - id INT IDENTITY PRIMARY KEY, - username NVARCHAR(256) NOT NULL UNIQUE, - password_hash NVARCHAR(512) NOT NULL, - created_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - last_used_at DATETIME2 NULL + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + username varchar(256) NOT NULL UNIQUE, + password_hash varchar(512) NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + last_used_at timestamptz NULL ); -- Seed a single default relay in the "Unconfigured" state: until an administrator picks a provider, -- Dispatch refuses to relay (mail is never silently delivered or discarded). INSERT INTO relays (name, provider, is_default, enabled, max_concurrency) -VALUES (N'default', N'Unconfigured', 1, 1, 4); +VALUES ('default', 'Unconfigured', true, true, 4); diff --git a/src/Dispatch.Data/Migrations/0002_relay_log_indexes.sql b/src/Dispatch.Data/Migrations/0002_relay_log_indexes.sql index fcefab0f..77c901d4 100644 --- a/src/Dispatch.Data/Migrations/0002_relay_log_indexes.sql +++ b/src/Dispatch.Data/Migrations/0002_relay_log_indexes.sql @@ -1,10 +1,8 @@ -- Spec §6.11: backing indexes for the Message Log "filter by relay" and "filter by routing rule" -- queries. Without these, those filters scan relay_log. Guarded so the migration is re-runnable. -IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_relay_log_relay' AND object_id = OBJECT_ID('relay_log')) - CREATE INDEX IX_relay_log_relay ON relay_log (relay_id, logged_at DESC) - INCLUDE (status, event, spool_id, from_address, to_domain, subject, provider, duration_ms); +CREATE INDEX IF NOT EXISTS IX_relay_log_relay ON relay_log (relay_id, logged_at DESC) + INCLUDE (status, event, spool_id, from_address, to_domain, subject, provider, duration_ms); -IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_relay_log_rule' AND object_id = OBJECT_ID('relay_log')) - CREATE INDEX IX_relay_log_rule ON relay_log (routing_rule_id, logged_at DESC) - INCLUDE (status, event, spool_id, from_address, to_domain, subject, provider, duration_ms); +CREATE INDEX IF NOT EXISTS IX_relay_log_rule ON relay_log (routing_rule_id, logged_at DESC) + INCLUDE (status, event, spool_id, from_address, to_domain, subject, provider, duration_ms); diff --git a/src/Dispatch.Data/Migrations/0003_drop_unconfigured_default.sql b/src/Dispatch.Data/Migrations/0003_drop_unconfigured_default.sql index 3088bb73..5b5abba5 100644 --- a/src/Dispatch.Data/Migrations/0003_drop_unconfigured_default.sql +++ b/src/Dispatch.Data/Migrations/0003_drop_unconfigured_default.sql @@ -8,8 +8,8 @@ -- degrades cleanly with no default: mail permanently fails with a clear "no provider configured" result -- until the wizard/first relay is added. DELETE FROM relays -WHERE provider = 'Unconfigured' AND is_default = 1 +WHERE provider = 'Unconfigured' AND is_default AND NOT EXISTS ( SELECT 1 FROM config c - WHERE c.[key] = N'relay:' + CAST(relays.id AS NVARCHAR(10)) + N':provider' + WHERE c."key" = 'relay:' || relays.id::text || ':provider' ); diff --git a/src/Dispatch.Data/Migrations/0004_relay_log_x_mailer_attachments.sql b/src/Dispatch.Data/Migrations/0004_relay_log_x_mailer_attachments.sql index a17d2eee..3f7f410c 100644 --- a/src/Dispatch.Data/Migrations/0004_relay_log_x_mailer_attachments.sql +++ b/src/Dispatch.Data/Migrations/0004_relay_log_x_mailer_attachments.sql @@ -2,8 +2,6 @@ -- header) and how many attachments the message carried. Captured at ingest and stored per relay_log row. -- Guarded so the migration is re-runnable. -IF COL_LENGTH('relay_log', 'x_mailer') IS NULL - ALTER TABLE relay_log ADD x_mailer NVARCHAR(256) NULL; +ALTER TABLE relay_log ADD COLUMN IF NOT EXISTS x_mailer varchar(256) NULL; -IF COL_LENGTH('relay_log', 'attachment_count') IS NULL - ALTER TABLE relay_log ADD attachment_count INT NOT NULL DEFAULT 0; +ALTER TABLE relay_log ADD COLUMN IF NOT EXISTS attachment_count int NOT NULL DEFAULT 0; diff --git a/src/Dispatch.Data/Migrations/0005_relay_log_perf_indexes.sql b/src/Dispatch.Data/Migrations/0005_relay_log_perf_indexes.sql index 03a1dadc..5a684965 100644 --- a/src/Dispatch.Data/Migrations/0005_relay_log_perf_indexes.sql +++ b/src/Dispatch.Data/Migrations/0005_relay_log_perf_indexes.sql @@ -6,7 +6,7 @@ -- composite (spool_id, logged_at, id) serves either direction with a seek instead of a scan. -- -- 2. Per-API-key message list (RecentByApiKeyAsync / GET /api/v1/messages) filters api_key_id and orders --- by logged_at DESC. A filtered index (api_key_id IS NOT NULL) keeps it tiny - the vast majority of +-- by logged_at DESC. A partial index (api_key_id IS NOT NULL) keeps it tiny - the vast majority of -- rows are SMTP ingest with a NULL api_key_id and are excluded from the index entirely. -- -- Other access paths are already covered: the unfiltered/date-range list orders by (logged_at DESC, id @@ -15,9 +15,7 @@ -- subject/tag use leading-wildcard LIKE and are inherently non-sargable (would need full-text), so no -- index is added for them. Guarded so the migration is re-runnable. -IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_relay_log_spool_id' AND object_id = OBJECT_ID('relay_log')) - CREATE INDEX IX_relay_log_spool_id ON relay_log (spool_id, logged_at, id); +CREATE INDEX IF NOT EXISTS IX_relay_log_spool_id ON relay_log (spool_id, logged_at, id); -IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_relay_log_api_key' AND object_id = OBJECT_ID('relay_log')) - CREATE INDEX IX_relay_log_api_key ON relay_log (api_key_id, logged_at DESC) - WHERE api_key_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS IX_relay_log_api_key ON relay_log (api_key_id, logged_at DESC) + WHERE api_key_id IS NOT NULL; diff --git a/src/Dispatch.Data/Migrations/0006_audit_log.sql b/src/Dispatch.Data/Migrations/0006_audit_log.sql index 4c6e136e..6629ee51 100644 --- a/src/Dispatch.Data/Migrations/0006_audit_log.sql +++ b/src/Dispatch.Data/Migrations/0006_audit_log.sql @@ -2,22 +2,19 @@ -- (auth, API-key changes, config changes) and unhandled server errors. `kind` is the coarse filter the -- UI exposes ('audit' vs 'error'); `category`/`severity` add detail. Guarded so the migration re-runs. -IF OBJECT_ID('audit_log', 'U') IS NULL -BEGIN - CREATE TABLE audit_log ( - id BIGINT IDENTITY PRIMARY KEY, - logged_at DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(), - kind NVARCHAR(16) NOT NULL, -- audit | error - category NVARCHAR(32) NOT NULL, -- Auth | ApiKey | Config | Error - event NVARCHAR(128) NOT NULL, - severity NVARCHAR(16) NOT NULL DEFAULT 'Info', -- Info | Notice | Warning | Error - actor NVARCHAR(128) NULL, - source_ip NVARCHAR(64) NULL, - detail NVARCHAR(MAX) NULL - ); +CREATE TABLE IF NOT EXISTS audit_log ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + logged_at timestamptz NOT NULL DEFAULT now(), + kind varchar(16) NOT NULL, -- audit | error + category varchar(32) NOT NULL, -- Auth | ApiKey | Config | Error + event varchar(128) NOT NULL, + severity varchar(16) NOT NULL DEFAULT 'Info', -- Info | Notice | Warning | Error + actor varchar(128) NULL, + source_ip varchar(64) NULL, + detail text NULL +); - -- Default listing order (newest first) + keyset tie-break. - CREATE INDEX IX_audit_log_at ON audit_log (logged_at DESC, id DESC); - -- The 'audit' vs 'error' filter. - CREATE INDEX IX_audit_log_kind ON audit_log (kind, logged_at DESC); -END +-- Default listing order (newest first) + keyset tie-break. +CREATE INDEX IF NOT EXISTS IX_audit_log_at ON audit_log (logged_at DESC, id DESC); +-- The 'audit' vs 'error' filter. +CREATE INDEX IF NOT EXISTS IX_audit_log_kind ON audit_log (kind, logged_at DESC); diff --git a/src/Dispatch.Data/Migrations/0007_relay_counters_system_bucket.sql b/src/Dispatch.Data/Migrations/0007_relay_counters_system_bucket.sql index 3ea1579f..bccec0b3 100644 --- a/src/Dispatch.Data/Migrations/0007_relay_counters_system_bucket.sql +++ b/src/Dispatch.Data/Migrations/0007_relay_counters_system_bucket.sql @@ -4,14 +4,7 @@ -- relay_log row (Message Log) recorded them. Drop the FK so relay_id = 0 is a valid "no specific relay" -- bucket. Aggregate summaries SUM across all rows (including 0); per-relay views join/filter to relay_id > 0, -- so the denied bucket never appears as a phantom relay. -DECLARE @fk sysname; -SELECT @fk = fk.name -FROM sys.foreign_keys fk -WHERE fk.parent_object_id = OBJECT_ID('dbo.relay_counters') - AND fk.referenced_object_id = OBJECT_ID('dbo.relays'); -IF @fk IS NOT NULL -BEGIN - -- EXEC() can't contain function calls inline, so build the statement into a variable first. - DECLARE @sql nvarchar(max) = N'ALTER TABLE dbo.relay_counters DROP CONSTRAINT ' + QUOTENAME(@fk); - EXEC sp_executesql @sql; -END +-- +-- The FK created inline in 0001 (relay_id int NOT NULL REFERENCES relays(id)) is auto-named by Postgres as +-- __fkey, so we drop it by that deterministic name (IF EXISTS makes this re-runnable). +ALTER TABLE relay_counters DROP CONSTRAINT IF EXISTS relay_counters_relay_id_fkey; diff --git a/src/Dispatch.Data/Repositories/SqlApiKeyRepository.cs b/src/Dispatch.Data/Repositories/SqlApiKeyRepository.cs index 6ad58b26..c424e1d3 100644 --- a/src/Dispatch.Data/Repositories/SqlApiKeyRepository.cs +++ b/src/Dispatch.Data/Repositories/SqlApiKeyRepository.cs @@ -31,11 +31,11 @@ public async Task CreateAsync(string name, int rateLimitPerMinute const string insert = """ INSERT INTO api_keys (key_id, key_hash, name, rate_limit_per_minute) - OUTPUT INSERTED.id, INSERTED.key_id AS KeyId, INSERTED.key_hash AS KeyHash, INSERTED.name, - INSERTED.created_at AS CreatedAt, INSERTED.last_used_at AS LastUsedAt, - INSERTED.message_count AS MessageCount, INSERTED.revoked, INSERTED.revoked_at AS RevokedAt, - INSERTED.rate_limit_per_minute AS RateLimitPerMinute - VALUES (@keyId, @hash, @name, @rateLimitPerMinute); + VALUES (@keyId, @hash, @name, @rateLimitPerMinute) + RETURNING id, key_id AS "KeyId", key_hash AS "KeyHash", name, + created_at AS "CreatedAt", last_used_at AS "LastUsedAt", + message_count AS "MessageCount", revoked, revoked_at AS "RevokedAt", + rate_limit_per_minute AS "RateLimitPerMinute"; """; await using var cn = await factory.OpenAsync(ct); @@ -47,7 +47,7 @@ INSERTED.rate_limit_per_minute AS RateLimitPerMinute public async Task> ListAsync(bool includeRevoked = false, CancellationToken ct = default) { - var where = includeRevoked ? "" : "WHERE revoked = 0"; + var where = includeRevoked ? "" : "WHERE NOT revoked"; await using var cn = await factory.OpenAsync(ct); var rows = await cn.QueryAsync(new CommandDefinition( $"SELECT {SelectColumns} FROM api_keys {where} ORDER BY created_at DESC", cancellationToken: ct)); @@ -58,7 +58,7 @@ public async Task RevokeAsync(int id, CancellationToken ct = default) { await using var cn = await factory.OpenAsync(ct); var affected = await cn.ExecuteAsync(new CommandDefinition( - "UPDATE api_keys SET revoked = 1, revoked_at = SYSUTCDATETIME() WHERE id = @id AND revoked = 0", + "UPDATE api_keys SET revoked = true, revoked_at = now() WHERE id = @id AND NOT revoked", new { id }, cancellationToken: ct)); return affected > 0; } @@ -74,7 +74,7 @@ public async Task RevokeAsync(int id, CancellationToken ct = default) var keyId = rawKey[..KeyIdLength]; await using var cn = await factory.OpenAsync(ct); var row = await cn.QuerySingleOrDefaultAsync(new CommandDefinition( - $"SELECT {SelectColumns} FROM api_keys WHERE key_id = @keyId AND revoked = 0", + $"SELECT {SelectColumns} FROM api_keys WHERE key_id = @keyId AND NOT revoked", new { keyId }, cancellationToken: ct)); if (row is null) @@ -90,7 +90,7 @@ public async Task RecordUsageAsync(int id, CancellationToken ct = default) { await using var cn = await factory.OpenAsync(ct); await cn.ExecuteAsync(new CommandDefinition( - "UPDATE api_keys SET last_used_at = SYSUTCDATETIME(), message_count = message_count + 1 WHERE id = @id", + "UPDATE api_keys SET last_used_at = now(), message_count = message_count + 1 WHERE id = @id", new { id }, cancellationToken: ct)); } diff --git a/src/Dispatch.Data/Repositories/SqlAuditLog.cs b/src/Dispatch.Data/Repositories/SqlAuditLog.cs index 439612c7..c6d8e1de 100644 --- a/src/Dispatch.Data/Repositories/SqlAuditLog.cs +++ b/src/Dispatch.Data/Repositories/SqlAuditLog.cs @@ -56,17 +56,18 @@ public async Task QueryAsync(AuditFilter filter, CancellationToken ct if (filter.Cursor is { } c) { where.Append(" AND (logged_at < @CursorAt OR (logged_at = @CursorAt AND id < @CursorId))"); - p.Add("CursorAt", c.LoggedAt, DbType.DateTime2); + p.Add("CursorAt", c.LoggedAt, DbType.DateTime); p.Add("CursorId", c.Id); } var sql = $""" - SELECT TOP (@Limit) + SELECT id AS Id, logged_at AS LoggedAt, kind AS Kind, category AS Category, event AS Event, severity AS Severity, actor AS Actor, source_ip AS SourceIp, detail AS Detail FROM audit_log {where} - ORDER BY logged_at DESC, id DESC; + ORDER BY logged_at DESC, id DESC + LIMIT @Limit; """; await using var cn = await factory.OpenAsync(ct); @@ -84,11 +85,11 @@ public async Task PurgeAsync(int generalRetentionDays, int securityRetentio // Noisy security events (allow-list denials, SMTP auth failures) are kept shorter. if (securityRetentionDays > 0) total += await DeleteBatchedAsync(cn, - "DELETE TOP (@Batch) FROM audit_log WHERE category IN ('Access','SmtpAuth') AND logged_at < DATEADD(DAY, -@Days, SYSUTCDATETIME());", + "DELETE FROM audit_log WHERE ctid IN (SELECT ctid FROM audit_log WHERE category IN ('Access','SmtpAuth') AND logged_at < now() - @Days * interval '1 day' LIMIT @Batch);", securityRetentionDays, ct); if (generalRetentionDays > 0) total += await DeleteBatchedAsync(cn, - "DELETE TOP (@Batch) FROM audit_log WHERE logged_at < DATEADD(DAY, -@Days, SYSUTCDATETIME());", + "DELETE FROM audit_log WHERE ctid IN (SELECT ctid FROM audit_log WHERE logged_at < now() - @Days * interval '1 day' LIMIT @Batch);", generalRetentionDays, ct); return total; } @@ -105,7 +106,7 @@ public async Task ArchiveAndDeleteOldestAsync(int batch, Dispatch.Core.Main { await using var cn = await factory.OpenAsync(ct); var rows = (await cn.QueryAsync(new CommandDefinition( - "SELECT TOP (@batch) * FROM audit_log ORDER BY logged_at ASC, id ASC;", + "SELECT * FROM audit_log ORDER BY logged_at ASC, id ASC LIMIT @batch;", new { batch }, cancellationToken: ct))).ToList(); if (rows.Count == 0) return 0; @@ -114,7 +115,7 @@ public async Task ArchiveAndDeleteOldestAsync(int batch, Dispatch.Core.Main var ids = rows.Select(r => Convert.ToInt64(((IDictionary)r)["id"])).ToArray(); return await cn.ExecuteAsync(new CommandDefinition( - "DELETE FROM audit_log WHERE id IN @ids;", new { ids }, cancellationToken: ct)); + "DELETE FROM audit_log WHERE id = ANY(@ids);", new { ids }, cancellationToken: ct)); } catch (Exception ex) { diff --git a/src/Dispatch.Data/Repositories/SqlConfigRepository.cs b/src/Dispatch.Data/Repositories/SqlConfigRepository.cs index 7e14d6a3..c1647ccf 100644 --- a/src/Dispatch.Data/Repositories/SqlConfigRepository.cs +++ b/src/Dispatch.Data/Repositories/SqlConfigRepository.cs @@ -14,7 +14,7 @@ public sealed class SqlConfigRepository(SqlConnectionFactory factory, ILogger( - new CommandDefinition("SELECT value AS Value, encrypted AS Encrypted FROM config WHERE [key] = @key", + new CommandDefinition("SELECT value AS Value, encrypted AS Encrypted FROM config WHERE \"key\" = @key", new { key }, cancellationToken: ct)); if (row is null) return null; return row.Value.Encrypted ? TryDecrypt(key, row.Value.Value) : row.Value.Value; @@ -24,10 +24,8 @@ public async Task SetAsync(string key, string value, bool encrypted = false, Can { var stored = encrypted ? SecureConfig.Encrypt(value) : value; const string sql = """ - MERGE config AS t - USING (VALUES (@key)) AS s([key]) ON t.[key] = s.[key] - WHEN MATCHED THEN UPDATE SET value = @stored, encrypted = @encrypted, updated_at = SYSUTCDATETIME() - WHEN NOT MATCHED THEN INSERT ([key], value, encrypted) VALUES (@key, @stored, @encrypted); + INSERT INTO config ("key", value, encrypted) VALUES (@key, @stored, @encrypted) + ON CONFLICT ("key") DO UPDATE SET value = @stored, encrypted = @encrypted, updated_at = now(); """; await using var cn = await factory.OpenAsync(ct); await cn.ExecuteAsync(new CommandDefinition(sql, new { key, stored, encrypted }, cancellationToken: ct)); @@ -37,7 +35,7 @@ public async Task> GetAllAsync(CancellationToken ct = { await using var cn = await factory.OpenAsync(ct); var rows = await cn.QueryAsync<(string Key, string Value, bool Encrypted, DateTime UpdatedAt)>( - new CommandDefinition("SELECT [key] AS [Key], value AS Value, encrypted AS Encrypted, updated_at AS UpdatedAt FROM config", + new CommandDefinition("SELECT \"key\" AS \"Key\", value AS Value, encrypted AS Encrypted, updated_at AS UpdatedAt FROM config", cancellationToken: ct)); return rows .Select(r => new ConfigEntry(r.Key, r.Encrypted ? TryDecrypt(r.Key, r.Value) ?? "" : r.Value, r.Encrypted, r.UpdatedAt)) diff --git a/src/Dispatch.Data/Repositories/SqlCounterRepository.cs b/src/Dispatch.Data/Repositories/SqlCounterRepository.cs index d057bdd0..f0e6d69b 100644 --- a/src/Dispatch.Data/Repositories/SqlCounterRepository.cs +++ b/src/Dispatch.Data/Repositories/SqlCounterRepository.cs @@ -26,11 +26,8 @@ public async Task IncrementAsync(int? relayId, CounterField field, CancellationT // Column name comes from a fixed enum mapping, never user input. var sql = $""" - MERGE relay_counters AS t - USING (VALUES (CAST(SYSUTCDATETIME() AS DATE), @relayId)) AS s(date, relay_id) - ON t.date = s.date AND t.relay_id = s.relay_id - WHEN MATCHED THEN UPDATE SET {column} = {column} + 1 - WHEN NOT MATCHED THEN INSERT (date, relay_id, {column}) VALUES (s.date, s.relay_id, 1); + INSERT INTO relay_counters (date, relay_id, {column}) VALUES (CURRENT_DATE, @relayId, 1) + ON CONFLICT (date, relay_id) DO UPDATE SET {column} = relay_counters.{column} + 1; """; await using var cn = await factory.OpenAsync(ct); @@ -41,13 +38,13 @@ public async Task GetTodayAsync(CancellationToken ct = default) { const string sql = """ SELECT - ISNULL(SUM(received), 0) AS Received, - ISNULL(SUM(delivered), 0) AS Delivered, - ISNULL(SUM(failed), 0) AS Failed, - ISNULL(SUM(retried), 0) AS Retried, - ISNULL(SUM(denied), 0) AS Denied + COALESCE(SUM(received), 0)::bigint AS Received, + COALESCE(SUM(delivered), 0)::bigint AS Delivered, + COALESCE(SUM(failed), 0)::bigint AS Failed, + COALESCE(SUM(retried), 0)::bigint AS Retried, + COALESCE(SUM(denied), 0)::bigint AS Denied FROM relay_counters - WHERE date = CAST(SYSUTCDATETIME() AS DATE); + WHERE date = CURRENT_DATE; """; await using var cn = await factory.OpenAsync(ct); return await cn.QuerySingleAsync(new CommandDefinition(sql, cancellationToken: ct)); @@ -57,13 +54,13 @@ public async Task> GetTodayByRelayAsync(Cancel { const string sql = """ SELECT relay_id AS RelayId, - ISNULL(SUM(received), 0) AS Received, - ISNULL(SUM(delivered), 0) AS Delivered, - ISNULL(SUM(failed), 0) AS Failed, - ISNULL(SUM(retried), 0) AS Retried, - ISNULL(SUM(denied), 0) AS Denied + COALESCE(SUM(received), 0)::bigint AS Received, + COALESCE(SUM(delivered), 0)::bigint AS Delivered, + COALESCE(SUM(failed), 0)::bigint AS Failed, + COALESCE(SUM(retried), 0)::bigint AS Retried, + COALESCE(SUM(denied), 0)::bigint AS Denied FROM relay_counters - WHERE date = CAST(SYSUTCDATETIME() AS DATE) + WHERE date = CURRENT_DATE AND relay_id > 0 -- exclude the relay_id 0 "no relay" bucket (denials) from per-relay views GROUP BY relay_id; """; @@ -76,11 +73,11 @@ public async Task GetRangeTotalsAsync(DateTime fromUtc, DateTime { const string sql = """ SELECT - ISNULL(SUM(received), 0) AS Received, - ISNULL(SUM(delivered), 0) AS Delivered, - ISNULL(SUM(failed), 0) AS Failed, - ISNULL(SUM(retried), 0) AS Retried, - ISNULL(SUM(denied), 0) AS Denied + COALESCE(SUM(received), 0)::bigint AS Received, + COALESCE(SUM(delivered), 0)::bigint AS Delivered, + COALESCE(SUM(failed), 0)::bigint AS Failed, + COALESCE(SUM(retried), 0)::bigint AS Retried, + COALESCE(SUM(denied), 0)::bigint AS Denied FROM relay_counters WHERE date >= @From AND date <= @To; """; @@ -91,12 +88,12 @@ FROM relay_counters public async Task> GetDailyAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct = default) { const string sql = """ - SELECT CONVERT(char(10), date, 23) AS Date, - ISNULL(SUM(received), 0) AS Received, - ISNULL(SUM(delivered), 0) AS Delivered, - ISNULL(SUM(failed), 0) AS Failed, - ISNULL(SUM(retried), 0) AS Retried, - ISNULL(SUM(denied), 0) AS Denied + SELECT to_char(date, 'YYYY-MM-DD') AS Date, + COALESCE(SUM(received), 0)::bigint AS Received, + COALESCE(SUM(delivered), 0)::bigint AS Delivered, + COALESCE(SUM(failed), 0)::bigint AS Failed, + COALESCE(SUM(retried), 0)::bigint AS Retried, + COALESCE(SUM(denied), 0)::bigint AS Denied FROM relay_counters WHERE date >= @From AND date <= @To GROUP BY date @@ -111,11 +108,11 @@ public async Task> GetRangeByRelayAsync(DateTime f { const string sql = """ SELECT c.relay_id AS RelayId, r.name AS RelayName, - ISNULL(SUM(c.received), 0) AS Received, - ISNULL(SUM(c.delivered), 0) AS Delivered, - ISNULL(SUM(c.failed), 0) AS Failed, - ISNULL(SUM(c.retried), 0) AS Retried, - ISNULL(SUM(c.denied), 0) AS Denied + COALESCE(SUM(c.received), 0)::bigint AS Received, + COALESCE(SUM(c.delivered), 0)::bigint AS Delivered, + COALESCE(SUM(c.failed), 0)::bigint AS Failed, + COALESCE(SUM(c.retried), 0)::bigint AS Retried, + COALESCE(SUM(c.denied), 0)::bigint AS Denied FROM relay_counters c JOIN relays r ON r.id = c.relay_id WHERE c.date >= @From AND c.date <= @To diff --git a/src/Dispatch.Data/Repositories/SqlDatabaseHealth.cs b/src/Dispatch.Data/Repositories/SqlDatabaseHealth.cs index a4b87155..2141fca1 100644 --- a/src/Dispatch.Data/Repositories/SqlDatabaseHealth.cs +++ b/src/Dispatch.Data/Repositories/SqlDatabaseHealth.cs @@ -1,6 +1,6 @@ using Dapper; using Dispatch.Core.Logging; -using Microsoft.Data.SqlClient; +using Npgsql; namespace Dispatch.Data.Repositories; @@ -10,12 +10,12 @@ public sealed class SqlDatabaseHealth(SqlConnectionFactory factory) : IDatabaseH public async Task IsReachableAsync(CancellationToken ct = default) { // Cap the connect string's connect timeout so a down database fails fast. - var cs = new SqlConnectionStringBuilder(factory.ConnectionString) { ConnectTimeout = 2 }.ConnectionString; + var cs = new NpgsqlConnectionStringBuilder(factory.ConnectionString) { Timeout = 2 }.ConnectionString; using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); cts.CancelAfter(TimeSpan.FromSeconds(3)); try { - await using var cn = new SqlConnection(cs); + await using var cn = new NpgsqlConnection(cs); await cn.OpenAsync(cts.Token); await cn.ExecuteScalarAsync(new CommandDefinition("SELECT 1", cancellationToken: cts.Token)); return true; diff --git a/src/Dispatch.Data/Repositories/SqlLogMaintenance.cs b/src/Dispatch.Data/Repositories/SqlLogMaintenance.cs index b1d07075..bd8d0872 100644 --- a/src/Dispatch.Data/Repositories/SqlLogMaintenance.cs +++ b/src/Dispatch.Data/Repositories/SqlLogMaintenance.cs @@ -11,9 +11,13 @@ public sealed class SqlLogMaintenance(SqlConnectionFactory factory) : ILogMainte public async Task PurgeByRetentionAsync(string @event, int retentionDays, CancellationToken ct = default) { + // Postgres has no DELETE ... LIMIT, so bound each batch by ctid membership. const string sql = """ - DELETE TOP (@BatchSize) FROM relay_log - WHERE event = @event AND logged_at < DATEADD(DAY, -@retentionDays, SYSUTCDATETIME()); + DELETE FROM relay_log + WHERE ctid IN ( + SELECT ctid FROM relay_log + WHERE event = @event AND logged_at < now() - @retentionDays * interval '1 day' + LIMIT @BatchSize); """; await using var cn = await factory.OpenAsync(ct); @@ -31,34 +35,26 @@ DELETE TOP (@BatchSize) FROM relay_log public async Task GetDatabaseSizeBytesAsync(CancellationToken ct = default) { - const string sql = "SELECT CAST(SUM(size) AS BIGINT) * 8 * 1024 FROM sys.database_files WHERE type = 0;"; + const string sql = "SELECT pg_database_size(current_database());"; await using var cn = await factory.OpenAsync(ct); return await cn.ExecuteScalarAsync(new CommandDefinition(sql, cancellationToken: ct)); } - public async Task GetDatabaseUsedBytesAsync(CancellationToken ct = default) + public async Task<(long TableBytes, long RowCount)> GetRelayLogStatsAsync(CancellationToken ct = default) { - // Used pages (not allocated file size) so the size-pressure loop terminates: this value drops as - // rows are deleted, whereas sys.database_files.size never shrinks on DELETE. - const string sql = "SELECT CAST(SUM(CAST(FILEPROPERTY(name, 'SpaceUsed') AS BIGINT)) AS BIGINT) * 8 * 1024 FROM sys.database_files WHERE type = 0;"; - await using var cn = await factory.OpenAsync(ct); - return await cn.ExecuteScalarAsync(new CommandDefinition(sql, cancellationToken: ct)); - } - - public async Task IsSizeCappedEditionAsync(CancellationToken ct = default) - { - // EngineEdition 4 = Express (the only edition with the 10 GB per-database data-file cap). 2/3 = - // Standard/Enterprise, 5/8 = Azure SQL DB/MI - none capped, so size-pressure must not run there. - const string sql = "SELECT CAST(SERVERPROPERTY('EngineEdition') AS INT);"; + const string sql = """ + SELECT pg_total_relation_size('relay_log')::bigint AS TableBytes, + (SELECT count(*) FROM relay_log)::bigint AS RowCount; + """; await using var cn = await factory.OpenAsync(ct); - return await cn.ExecuteScalarAsync(new CommandDefinition(sql, cancellationToken: ct)) == 4; + return await cn.QuerySingleAsync<(long, long)>(new CommandDefinition(sql, cancellationToken: ct)); } public async Task ArchiveAndDeleteOldestRelayLogAsync(int batch, ArchiveRows archive, CancellationToken ct = default) { await using var cn = await factory.OpenAsync(ct); var rows = (await cn.QueryAsync(new CommandDefinition( - "SELECT TOP (@batch) * FROM relay_log ORDER BY logged_at ASC, id ASC;", + "SELECT * FROM relay_log ORDER BY logged_at ASC, id ASC LIMIT @batch;", new { batch }, cancellationToken: ct))).ToList(); if (rows.Count == 0) return 0; @@ -67,7 +63,16 @@ public async Task ArchiveAndDeleteOldestRelayLogAsync(int batch, ArchiveRow var ids = rows.Select(r => Convert.ToInt64(((IDictionary)r)["id"])).ToArray(); return await cn.ExecuteAsync(new CommandDefinition( - "DELETE FROM relay_log WHERE id IN @ids;", new { ids }, cancellationToken: ct)); + "DELETE FROM relay_log WHERE id = ANY(@ids);", new { ids }, cancellationToken: ct)); + } + + public async Task VacuumLogTablesAsync(CancellationToken ct = default) + { + // VACUUM cannot run inside a transaction block; each statement runs on its own. FULL rewrites the + // table and returns space to the OS so pg_database_size drops and the size-pressure trigger clears. + await using var cn = await factory.OpenAsync(ct); + await cn.ExecuteAsync(new CommandDefinition("VACUUM (FULL) relay_log;", cancellationToken: ct)); + await cn.ExecuteAsync(new CommandDefinition("VACUUM (FULL) audit_log;", cancellationToken: ct)); } // Materializes a Dapper row into a nullable-friendly read-only dictionary for archiving/serialization. diff --git a/src/Dispatch.Data/Repositories/SqlMessageLogQuery.cs b/src/Dispatch.Data/Repositories/SqlMessageLogQuery.cs index e251c8cc..33c81885 100644 --- a/src/Dispatch.Data/Repositories/SqlMessageLogQuery.cs +++ b/src/Dispatch.Data/Repositories/SqlMessageLogQuery.cs @@ -24,16 +24,17 @@ public async Task QueryAsync(MessageLogFilter filter, Cancellati if (filter.Cursor is { } cursor) { where.Append(" AND (logged_at < @CursorLoggedAt OR (logged_at = @CursorLoggedAt AND id < @CursorId))"); - p.Add("CursorLoggedAt", cursor.LoggedAt, DbType.DateTime2); + p.Add("CursorLoggedAt", cursor.LoggedAt, DbType.DateTime); p.Add("CursorId", cursor.Id); } var sql = $""" - SELECT TOP (@Limit) + SELECT {RowColumns} FROM relay_log {where} - ORDER BY logged_at DESC, id DESC; + ORDER BY logged_at DESC, id DESC + LIMIT @Limit; """; await using var cn = await factory.OpenAsync(ct); @@ -70,7 +71,7 @@ FROM relay_log ) t WHERE rn = 1 ORDER BY logged_at DESC, id DESC - OFFSET @Offset ROWS FETCH NEXT @Limit ROWS ONLY; + LIMIT @Limit OFFSET @Offset; """; await using var cn = await factory.OpenAsync(ct); @@ -95,11 +96,11 @@ FROM relay_log // Shared filter clauses (no cursor/paging). Every user value is a parameter - never interpolated. private static void AppendFilters(StringBuilder where, DynamicParameters p, MessageLogFilter filter) { - // Bind dates as datetime2 to match the column precision. - if (filter.FromUtc is { } from) { where.Append(" AND logged_at >= @FromUtc"); p.Add("FromUtc", from, DbType.DateTime2); } - if (filter.ToUtc is { } to) { where.Append(" AND logged_at < @ToUtc"); p.Add("ToUtc", to, DbType.DateTime2); } - if (filter.Statuses is { Length: > 0 } s) { where.Append(" AND status IN @Statuses"); p.Add("Statuses", s); } - if (filter.Events is { Length: > 0 } ev) { where.Append(" AND event IN @Events"); p.Add("Events", ev); } + // Bind dates as timestamp to match the timestamptz column type. + if (filter.FromUtc is { } from) { where.Append(" AND logged_at >= @FromUtc"); p.Add("FromUtc", from, DbType.DateTime); } + if (filter.ToUtc is { } to) { where.Append(" AND logged_at < @ToUtc"); p.Add("ToUtc", to, DbType.DateTime); } + if (filter.Statuses is { Length: > 0 } s) { where.Append(" AND status = ANY(@Statuses)"); p.Add("Statuses", s); } + if (filter.Events is { Length: > 0 } ev) { where.Append(" AND event = ANY(@Events)"); p.Add("Events", ev); } if (!string.IsNullOrWhiteSpace(filter.IngestSource)) { where.Append(" AND ingest_source = @IngestSource"); p.Add("IngestSource", filter.IngestSource); } if (!string.IsNullOrWhiteSpace(filter.FromDomain)) { where.Append(" AND from_domain = @FromDomain"); p.Add("FromDomain", filter.FromDomain); } if (!string.IsNullOrWhiteSpace(filter.ToDomain)) { where.Append(" AND to_domain = @ToDomain"); p.Add("ToDomain", filter.ToDomain); } @@ -127,14 +128,15 @@ private static void AppendFilters(StringBuilder where, DynamicParameters p, Mess public async Task GetBySpoolIdAsync(string spoolId, int? apiKeyId, CancellationToken ct = default) { const string sql = """ - SELECT TOP 1 + SELECT id AS Id, logged_at AS LoggedAt, event AS Event, status AS Status, spool_id AS SpoolId, from_address AS FromAddress, to_domain AS ToDomain, subject AS Subject, relay_name AS RelayName, provider AS Provider, duration_ms AS DurationMs, size_bytes AS SizeBytes, ingest_source AS IngestSource, retry_attempt AS RetryAttempt, error AS Error FROM relay_log WHERE spool_id = @spoolId AND (@apiKeyId IS NULL OR api_key_id = @apiKeyId) - ORDER BY logged_at DESC, id DESC; + ORDER BY logged_at DESC, id DESC + LIMIT 1; """; await using var cn = await factory.OpenAsync(ct); return await cn.QuerySingleOrDefaultAsync( @@ -149,18 +151,19 @@ public async Task> RecentByApiKeyAsync( var p = new DynamicParameters(); p.Add("ApiKeyId", apiKeyId); p.Add("Limit", limit); - if (statuses is { Length: > 0 }) { where.Append(" AND status IN @Statuses"); p.Add("Statuses", statuses); } + if (statuses is { Length: > 0 }) { where.Append(" AND status = ANY(@Statuses)"); p.Add("Statuses", statuses); } // One row per message (latest event), matching the dashboard list - see GroupKey. var sql = $""" - SELECT TOP (@Limit) {RowColumns} + SELECT {RowColumns} FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY {GroupKey} ORDER BY logged_at DESC, id DESC) AS rn FROM relay_log {where} ) t WHERE rn = 1 - ORDER BY logged_at DESC, id DESC; + ORDER BY logged_at DESC, id DESC + LIMIT @Limit; """; await using var cn = await factory.OpenAsync(ct); return (await cn.QueryAsync(new CommandDefinition(sql, p, cancellationToken: ct))).ToList(); @@ -169,7 +172,7 @@ FROM relay_log public async Task GetByIdAsync(long id, CancellationToken ct = default) { const string sql = """ - SELECT TOP 1 + SELECT id AS Id, logged_at AS LoggedAt, event AS Event, status AS Status, spool_id AS SpoolId, retry_attempt AS RetryAttempt, from_address AS FromAddress, from_domain AS FromDomain, to_addresses AS ToAddressesJson, to_domain AS ToDomain, subject AS Subject, size_bytes AS SizeBytes, @@ -178,7 +181,8 @@ SELECT TOP 1 duration_ms AS DurationMs, error AS Error, ingest_source AS IngestSource, source_ip AS SourceIp, api_key_name AS ApiKeyName, tags AS TagsJson, x_mailer AS XMailer, attachment_count AS AttachmentCount FROM relay_log - WHERE id = @id; + WHERE id = @id + LIMIT 1; """; await using var cn = await factory.OpenAsync(ct); var raw = await cn.QuerySingleOrDefaultAsync( diff --git a/src/Dispatch.Data/Repositories/SqlRelayRepository.cs b/src/Dispatch.Data/Repositories/SqlRelayRepository.cs index 8cba191b..dd61646b 100644 --- a/src/Dispatch.Data/Repositories/SqlRelayRepository.cs +++ b/src/Dispatch.Data/Repositories/SqlRelayRepository.cs @@ -14,7 +14,7 @@ public sealed class SqlRelayRepository(SqlConnectionFactory factory) : IRelayRep private const string SelectColumns = "id, name, provider, is_default AS IsDefault, enabled, max_concurrency AS MaxConcurrency, max_message_bytes AS MaxMessageBytes"; private const string InsertedColumns = - "INSERTED.id, INSERTED.name, INSERTED.provider, INSERTED.is_default AS IsDefault, INSERTED.enabled, INSERTED.max_concurrency AS MaxConcurrency, INSERTED.max_message_bytes AS MaxMessageBytes"; + "id, name, provider, is_default AS \"IsDefault\", enabled, max_concurrency AS \"MaxConcurrency\", max_message_bytes AS \"MaxMessageBytes\""; private readonly Lock _lock = new(); private RelayRecord? _cachedDefault; @@ -30,7 +30,7 @@ public sealed class SqlRelayRepository(SqlConnectionFactory factory) : IRelayRep await using var cn = await factory.OpenAsync(ct); var row = await cn.QuerySingleOrDefaultAsync(new CommandDefinition( - $"SELECT TOP 1 {SelectColumns} FROM relays WHERE is_default = 1 AND enabled = 1", cancellationToken: ct)); + $"SELECT {SelectColumns} FROM relays WHERE is_default AND enabled LIMIT 1", cancellationToken: ct)); var record = row?.ToRecord(); lock (_lock) { _cachedDefault = record; _cachedAtUtc = DateTime.UtcNow; } @@ -60,9 +60,9 @@ public async Task CreateAsync( // single-provider setup "just works" with no extra step. Subsequent relays are non-default. const string sql = $""" INSERT INTO relays (name, provider, max_concurrency, max_message_bytes, is_default) - OUTPUT {InsertedColumns} VALUES (@name, @provider, @maxConcurrency, @maxMessageBytes, - CASE WHEN EXISTS (SELECT 1 FROM relays WHERE is_default = 1) THEN 0 ELSE 1 END); + CASE WHEN EXISTS (SELECT 1 FROM relays WHERE is_default) THEN false ELSE true END) + RETURNING {InsertedColumns}; """; await using var cn = await factory.OpenAsync(ct); var row = await cn.QuerySingleAsync(new CommandDefinition( @@ -76,7 +76,7 @@ public async Task UpdateAsync( { const string sql = """ UPDATE relays SET name = @name, provider = @provider, enabled = @enabled, max_concurrency = @maxConcurrency, - max_message_bytes = @maxMessageBytes, updated_at = SYSUTCDATETIME() + max_message_bytes = @maxMessageBytes, updated_at = now() WHERE id = @id; """; await using var cn = await factory.OpenAsync(ct); @@ -95,8 +95,8 @@ public async Task DeleteAsync(int id, CancellationToken ct = default) // Preserve log history (relay_name is denormalised) but clear the FK; drop counters + credentials. await cn.ExecuteAsync(new CommandDefinition("UPDATE relay_log SET relay_id = NULL WHERE relay_id = @id", new { id }, tx, cancellationToken: ct)); await cn.ExecuteAsync(new CommandDefinition("DELETE FROM relay_counters WHERE relay_id = @id", new { id }, tx, cancellationToken: ct)); - await cn.ExecuteAsync(new CommandDefinition("DELETE FROM config WHERE [key] LIKE @prefix", new { prefix = $"relay:{id}:%" }, tx, cancellationToken: ct)); - var n = await cn.ExecuteAsync(new CommandDefinition("DELETE FROM relays WHERE id = @id AND is_default = 0", new { id }, tx, cancellationToken: ct)); + await cn.ExecuteAsync(new CommandDefinition("DELETE FROM config WHERE \"key\" LIKE @prefix", new { prefix = $"relay:{id}:%" }, tx, cancellationToken: ct)); + var n = await cn.ExecuteAsync(new CommandDefinition("DELETE FROM relays WHERE id = @id AND NOT is_default", new { id }, tx, cancellationToken: ct)); await tx.CommitAsync(ct); InvalidateCache(); return n > 0; @@ -118,8 +118,8 @@ public async Task SetDefaultAsync(int id, CancellationToken ct = default) "SELECT 1 FROM relays WHERE id = @id", new { id }, tx, cancellationToken: ct)); if (exists is null) { await tx.RollbackAsync(ct); return false; } - await cn.ExecuteAsync(new CommandDefinition("UPDATE relays SET is_default = 0 WHERE is_default = 1", transaction: tx, cancellationToken: ct)); - await cn.ExecuteAsync(new CommandDefinition("UPDATE relays SET is_default = 1, enabled = 1 WHERE id = @id", new { id }, tx, cancellationToken: ct)); + await cn.ExecuteAsync(new CommandDefinition("UPDATE relays SET is_default = false WHERE is_default", transaction: tx, cancellationToken: ct)); + await cn.ExecuteAsync(new CommandDefinition("UPDATE relays SET is_default = true, enabled = true WHERE id = @id", new { id }, tx, cancellationToken: ct)); await tx.CommitAsync(ct); InvalidateCache(); return true; diff --git a/src/Dispatch.Data/Repositories/SqlRoutingRuleRepository.cs b/src/Dispatch.Data/Repositories/SqlRoutingRuleRepository.cs index 877a96b7..283f6f69 100644 --- a/src/Dispatch.Data/Repositories/SqlRoutingRuleRepository.cs +++ b/src/Dispatch.Data/Repositories/SqlRoutingRuleRepository.cs @@ -9,13 +9,13 @@ public sealed class SqlRoutingRuleRepository(SqlConnectionFactory factory) : IRo private const string SelectColumns = "id, priority, name, recipient_pattern AS RecipientPattern, sender_pattern AS SenderPattern, relay_id AS RelayId, enabled"; private const string InsertedColumns = - "INSERTED.id, INSERTED.priority, INSERTED.name, INSERTED.recipient_pattern AS RecipientPattern, INSERTED.sender_pattern AS SenderPattern, INSERTED.relay_id AS RelayId, INSERTED.enabled"; + "id, priority, name, recipient_pattern AS \"RecipientPattern\", sender_pattern AS \"SenderPattern\", relay_id AS \"RelayId\", enabled"; public async Task> GetEnabledOrderedAsync(CancellationToken ct = default) { await using var cn = await factory.OpenAsync(ct); var rows = await cn.QueryAsync(new CommandDefinition( - $"SELECT {SelectColumns} FROM routing_rules WHERE enabled = 1", cancellationToken: ct)); + $"SELECT {SelectColumns} FROM routing_rules WHERE enabled", cancellationToken: ct)); // Specificity is computed in code (spec §10.5: priority ASC, then specificity DESC). return rows.Select(r => r.ToRule()) .OrderBy(r => r.Priority) @@ -41,8 +41,8 @@ public async Task CreateAsync(RoutingRule rule, CancellationToken c const string sql = $""" INSERT INTO routing_rules (priority, name, recipient_pattern, sender_pattern, relay_id, enabled) - OUTPUT {InsertedColumns} - VALUES (@priority, @Name, @RecipientPattern, @SenderPattern, @RelayId, @Enabled); + VALUES (@priority, @Name, @RecipientPattern, @SenderPattern, @RelayId, @Enabled) + RETURNING {InsertedColumns}; """; var row = await cn.QuerySingleAsync(new CommandDefinition(sql, new { diff --git a/src/Dispatch.Data/Repositories/SqlSmtpCredentialRepository.cs b/src/Dispatch.Data/Repositories/SqlSmtpCredentialRepository.cs index 42ea798c..dd15ca70 100644 --- a/src/Dispatch.Data/Repositories/SqlSmtpCredentialRepository.cs +++ b/src/Dispatch.Data/Repositories/SqlSmtpCredentialRepository.cs @@ -23,7 +23,7 @@ public async Task VerifyAsync(string username, string password, Cancellati if (!BCrypt.Net.BCrypt.Verify(password, hash)) return false; await cn.ExecuteAsync(new CommandDefinition( - "UPDATE config_smtp_credentials SET last_used_at = SYSUTCDATETIME() WHERE username = @username", + "UPDATE config_smtp_credentials SET last_used_at = now() WHERE username = @username", new { username }, cancellationToken: ct)); return true; } @@ -41,10 +41,8 @@ public async Task AddAsync(string username, string password, CancellationToken c { var hash = BCrypt.Net.BCrypt.HashPassword(password, WorkFactor); const string sql = """ - MERGE config_smtp_credentials AS t - USING (VALUES (@username)) AS s(username) ON t.username = s.username - WHEN MATCHED THEN UPDATE SET password_hash = @hash - WHEN NOT MATCHED THEN INSERT (username, password_hash) VALUES (@username, @hash); + INSERT INTO config_smtp_credentials (username, password_hash) VALUES (@username, @hash) + ON CONFLICT (username) DO UPDATE SET password_hash = @hash; """; await using var cn = await factory.OpenAsync(ct); await cn.ExecuteAsync(new CommandDefinition(sql, new { username, hash }, cancellationToken: ct)); diff --git a/src/Dispatch.Data/Repositories/SqlStorageReport.cs b/src/Dispatch.Data/Repositories/SqlStorageReport.cs index 04dcdd92..579ad1f3 100644 --- a/src/Dispatch.Data/Repositories/SqlStorageReport.cs +++ b/src/Dispatch.Data/Repositories/SqlStorageReport.cs @@ -5,8 +5,8 @@ namespace Dispatch.Data.Repositories; /// /// Computes database-side storage usage (spec §6.10): per-event relay_log row counts, the relay_log and -/// audit_log table sizes, and the audit row counts. Table sizes come from sys.allocation_units (the same -/// figures sp_spaceused reports). Best-effort: if SQL is unreachable, returns a not-connected snapshot +/// audit_log table sizes, and the audit row counts. Table sizes come from pg_total_relation_size +/// (data + indexes + TOAST). Best-effort: if the database is unreachable, returns a not-connected snapshot /// rather than throwing, so the dashboard storage view degrades gracefully. /// public sealed class SqlStorageReport(SqlConnectionFactory factory) : IStorageReport @@ -18,11 +18,11 @@ public async Task GetAsync(CancellationToken ct = default) await using var cn = await factory.OpenAsync(ct); var dbBytes = await cn.ExecuteScalarAsync(new CommandDefinition( - "SELECT CAST(SUM(size) AS BIGINT) * 8 * 1024 FROM sys.database_files WHERE type = 0;", + "SELECT pg_database_size(current_database());", cancellationToken: ct)); var byEvent = (await cn.QueryAsync(new CommandDefinition( - "SELECT event AS [Event], COUNT_BIG(*) AS Rows FROM relay_log GROUP BY event;", + "SELECT event AS \"Event\", count(*) AS \"Rows\" FROM relay_log GROUP BY event;", cancellationToken: ct))).ToList(); var relayLogBytes = await TableBytesAsync(cn, "relay_log", ct); @@ -30,8 +30,8 @@ public async Task GetAsync(CancellationToken ct = default) var audit = await cn.QuerySingleAsync<(long Total, long Security)>(new CommandDefinition( """ - SELECT COUNT_BIG(*) AS Total, - ISNULL(SUM(CASE WHEN category IN ('Access','SmtpAuth') THEN 1 ELSE 0 END), 0) AS Security + SELECT count(*) AS Total, + COALESCE(SUM(CASE WHEN category IN ('Access','SmtpAuth') THEN 1 ELSE 0 END), 0) AS Security FROM audit_log; """, cancellationToken: ct)); @@ -44,16 +44,10 @@ SELECT COUNT_BIG(*) AS Total, } } - // Total allocated bytes for a table (data + indexes), from the allocation-unit page counts (8 KB pages). + // Total on-disk bytes for a table (data + indexes + TOAST). Passing the name through to_regclass keeps + // the lookup safe and returns NULL (→ 0) if the table doesn't exist yet. private static async Task TableBytesAsync(System.Data.Common.DbConnection cn, string table, CancellationToken ct) => await cn.ExecuteScalarAsync(new CommandDefinition( - """ - SELECT ISNULL(SUM(a.total_pages), 0) * 8 * 1024 - FROM sys.tables t - JOIN sys.indexes i ON t.object_id = i.object_id - JOIN sys.partitions p ON i.object_id = p.object_id AND i.index_id = p.index_id - JOIN sys.allocation_units a ON p.partition_id = a.container_id - WHERE t.name = @table; - """, + "SELECT COALESCE(pg_total_relation_size(to_regclass(@table)), 0)::bigint;", new { table }, cancellationToken: ct)); } diff --git a/src/Dispatch.Data/SqlConnectionFactory.cs b/src/Dispatch.Data/SqlConnectionFactory.cs index 48592999..e2fbeaa4 100644 --- a/src/Dispatch.Data/SqlConnectionFactory.cs +++ b/src/Dispatch.Data/SqlConnectionFactory.cs @@ -1,17 +1,17 @@ -using Microsoft.Data.SqlClient; +using Npgsql; namespace Dispatch.Data; -/// Creates SQL connections from the configured connection string. +/// Creates PostgreSQL connections from the configured connection string. public sealed class SqlConnectionFactory(string connectionString) { public string ConnectionString { get; } = connectionString; - public SqlConnection Create() => new(ConnectionString); + public NpgsqlConnection Create() => new(ConnectionString); - public async Task OpenAsync(CancellationToken ct = default) + public async Task OpenAsync(CancellationToken ct = default) { - var cn = new SqlConnection(ConnectionString); + var cn = new NpgsqlConnection(ConnectionString); await cn.OpenAsync(ct); return cn; } diff --git a/src/Dispatch.Service/appsettings.json b/src/Dispatch.Service/appsettings.json index 0ce400bd..31f94d74 100644 --- a/src/Dispatch.Service/appsettings.json +++ b/src/Dispatch.Service/appsettings.json @@ -1,5 +1,5 @@ { - "_comment": "Spec §12.1: appsettings holds ONLY the DB connection string and the Web UI TLS cert. All other settings live in the SQL config table and are managed from the dashboard. The installer writes ConnectionStrings:DispatchLog; dev uses appsettings.Development.json.", + "_comment": "Spec §12.1: appsettings holds ONLY the PostgreSQL connection string and the Web UI TLS cert. All other settings live in the config table and are managed from the dashboard. The installer writes ConnectionStrings:DispatchLog; dev uses appsettings.Development.json.", "ConnectionStrings": { "DispatchLog": "" }, diff --git a/src/Dispatch.Web/Endpoints/WebEndpoints.cs b/src/Dispatch.Web/Endpoints/WebEndpoints.cs index 9b128214..d2652db8 100644 --- a/src/Dispatch.Web/Endpoints/WebEndpoints.cs +++ b/src/Dispatch.Web/Endpoints/WebEndpoints.cs @@ -124,7 +124,7 @@ public static void MapDashboardApi(this IEndpointRouteBuilder app, int webPort) var message = suspended ? "Disk space critically low - SMTP intake suspended" : connected ? null - : "SQL Server unavailable - mail flow unaffected; UI log unavailable"; + : "Database unavailable - mail flow unaffected; UI log unavailable"; var payload = new { @@ -226,7 +226,7 @@ public static void MapDashboardApi(this IEndpointRouteBuilder app, int webPort) // Storage usage broken out by retention category (spec §6.10): how much each log-event class, the // audit log, and each spool dir is actually consuming - so operators can see what their retention // windows hold. DB per-event bytes are estimated by apportioning the relay_log table size across the - // event row counts (exact per-event bytes aren't cheap in SQL Server); spool figures are exact. + // event row counts (exact per-event bytes aren't cheap in PostgreSQL); spool figures are exact. group.MapGet("/storage", async (IStorageReport report, SpoolDirectory spool, CancellationToken ct) => { var db = await report.GetAsync(ct); @@ -432,7 +432,7 @@ internal static RelayLogEntry TestEntry( ProviderResponse = detail, Error = error, DurationMs = durationMs, - IngestSource = "Dashboard test", // distinguishes a dashboard "Send test" from real SMTP/API traffic (NVARCHAR(16)) + IngestSource = "Dashboard test", // distinguishes a dashboard "Send test" from real SMTP/API traffic (varchar(16)) }; public sealed record CreateKeyRequest(string Name, int? RateLimitPerMinute); diff --git a/tests/Dispatch.Core.Tests/PurgeWorkerTests.cs b/tests/Dispatch.Core.Tests/PurgeWorkerTests.cs index f0bb0c5c..9f1357b7 100644 --- a/tests/Dispatch.Core.Tests/PurgeWorkerTests.cs +++ b/tests/Dispatch.Core.Tests/PurgeWorkerTests.cs @@ -12,9 +12,9 @@ private sealed class NoopLogMaintenance : ILogMaintenance { public Task PurgeByRetentionAsync(string @event, int retentionDays, CancellationToken ct = default) => Task.FromResult(0); public Task GetDatabaseSizeBytesAsync(CancellationToken ct = default) => Task.FromResult(0L); - public Task GetDatabaseUsedBytesAsync(CancellationToken ct = default) => Task.FromResult(0L); - public Task IsSizeCappedEditionAsync(CancellationToken ct = default) => Task.FromResult(true); + public Task<(long TableBytes, long RowCount)> GetRelayLogStatsAsync(CancellationToken ct = default) => Task.FromResult((0L, 0L)); public Task ArchiveAndDeleteOldestRelayLogAsync(int batch, ArchiveRows archive, CancellationToken ct = default) => Task.FromResult(0); + public Task VacuumLogTablesAsync(CancellationToken ct = default) => Task.CompletedTask; } private sealed class RecordingLogMaintenance : ILogMaintenance @@ -23,22 +23,22 @@ private sealed class RecordingLogMaintenance : ILogMaintenance public Task PurgeByRetentionAsync(string @event, int retentionDays, CancellationToken ct = default) { RetentionPurgedEvents.Add(@event); return Task.FromResult(0); } public Task GetDatabaseSizeBytesAsync(CancellationToken ct = default) => Task.FromResult(0L); - public Task GetDatabaseUsedBytesAsync(CancellationToken ct = default) => Task.FromResult(0L); - public Task IsSizeCappedEditionAsync(CancellationToken ct = default) => Task.FromResult(true); + public Task<(long TableBytes, long RowCount)> GetRelayLogStatsAsync(CancellationToken ct = default) => Task.FromResult((0L, 0L)); public Task ArchiveAndDeleteOldestRelayLogAsync(int batch, ArchiveRows archive, CancellationToken ct = default) => Task.FromResult(0); + public Task VacuumLogTablesAsync(CancellationToken ct = default) => Task.CompletedTask; } - // Records size-pressure activity and lets the test pick the SQL edition. Reports 50 GB used (over any - // trigger) and deletes nothing, so the loop makes exactly one archive-and-delete attempt then stops. - private sealed class SizePressureSpy(bool capped) : ILogMaintenance + // Records size-pressure activity. Reports 50 GB (over any configured trigger) with a plausible relay_log + // size/rowcount, and deletes nothing, so the loop makes exactly one archive-and-delete attempt then stops. + private sealed class SizePressureSpy : ILogMaintenance { public int ArchiveAndDeleteCalls { get; private set; } public Task PurgeByRetentionAsync(string @event, int retentionDays, CancellationToken ct = default) => Task.FromResult(0); public Task GetDatabaseSizeBytesAsync(CancellationToken ct = default) => Task.FromResult(50L * 1024 * 1024 * 1024); - public Task GetDatabaseUsedBytesAsync(CancellationToken ct = default) => Task.FromResult(50L * 1024 * 1024 * 1024); - public Task IsSizeCappedEditionAsync(CancellationToken ct = default) => Task.FromResult(capped); + public Task<(long TableBytes, long RowCount)> GetRelayLogStatsAsync(CancellationToken ct = default) => Task.FromResult((40L * 1024 * 1024 * 1024, 1_000_000L)); public Task ArchiveAndDeleteOldestRelayLogAsync(int batch, ArchiveRows archive, CancellationToken ct = default) { ArchiveAndDeleteCalls++; return Task.FromResult(0); } + public Task VacuumLogTablesAsync(CancellationToken ct = default) => Task.CompletedTask; } [Fact] @@ -152,22 +152,23 @@ public async Task Archive_files_pruned_by_retention() } [Fact] - public async Task Size_pressure_runs_only_on_express() + public async Task Size_pressure_runs_only_when_trigger_configured() { using var t = new TempSpool(); var disk = new DiskMonitor(t.Spool, new IntakeState(), _ => long.MaxValue, NullLogger.Instance); - var opts = new PurgeOptions { SizePressure = new PurgeOptions.SizePressureOptions { TriggerGb = 9.5, TargetGb = 9.0 } }; - - // Express edition + 50 GB > 9.5 GB trigger → size-pressure archives + purges the oldest rows. - var express = new SizePressureSpy(capped: true); - await new PurgeWorker(t.Spool, express, disk, new OptionsPurgeSettings(opts), new PurgeHistory(), NullLogger.Instance) - .RunOnceAsync(opts, default); - Assert.True(express.ArchiveAndDeleteCalls >= 1); - - // Non-Express (no 10 GB cap) → size-pressure is skipped even at 50 GB. - var standard = new SizePressureSpy(capped: false); - await new PurgeWorker(t.Spool, standard, disk, new OptionsPurgeSettings(opts), new PurgeHistory(), NullLogger.Instance) - .RunOnceAsync(opts, default); - Assert.Equal(0, standard.ArchiveAndDeleteCalls); + + // Trigger configured (9.5 GB) + 50 GB DB > trigger → size-pressure archives + purges the oldest rows. + var enabledOpts = new PurgeOptions { SizePressure = new PurgeOptions.SizePressureOptions { TriggerGb = 9.5, TargetGb = 9.0 } }; + var triggered = new SizePressureSpy(); + await new PurgeWorker(t.Spool, triggered, disk, new OptionsPurgeSettings(enabledOpts), new PurgeHistory(), NullLogger.Instance) + .RunOnceAsync(enabledOpts, default); + Assert.True(triggered.ArchiveAndDeleteCalls >= 1); + + // Trigger = 0 (the default: Postgres has no hard cap) → size-pressure is skipped even at 50 GB. + var disabledOpts = new PurgeOptions { SizePressure = new PurgeOptions.SizePressureOptions { TriggerGb = 0, TargetGb = 0 } }; + var disabled = new SizePressureSpy(); + await new PurgeWorker(t.Spool, disabled, disk, new OptionsPurgeSettings(disabledOpts), new PurgeHistory(), NullLogger.Instance) + .RunOnceAsync(disabledOpts, default); + Assert.Equal(0, disabled.ArchiveAndDeleteCalls); } } diff --git a/tests/Dispatch.Data.Tests/Dispatch.Data.Tests.csproj b/tests/Dispatch.Data.Tests/Dispatch.Data.Tests.csproj index a1c502f7..d7df0e69 100644 --- a/tests/Dispatch.Data.Tests/Dispatch.Data.Tests.csproj +++ b/tests/Dispatch.Data.Tests/Dispatch.Data.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Dispatch.Data.Tests/SqlAuditLogTests.cs b/tests/Dispatch.Data.Tests/SqlAuditLogTests.cs index 82c90f2f..209f9479 100644 --- a/tests/Dispatch.Data.Tests/SqlAuditLogTests.cs +++ b/tests/Dispatch.Data.Tests/SqlAuditLogTests.cs @@ -41,7 +41,7 @@ public async Task Purge_respects_general_and_security_retention() // fresh one. Dapper-parameterised inserts mirroring the table shape. async Task Seed(string kind, string category, int daysOld) => await Dapper.SqlMapper.ExecuteAsync(cn, - "INSERT INTO audit_log (logged_at, kind, category, event, severity) VALUES (DATEADD(DAY, -@d, SYSUTCDATETIME()), @k, @c, 'x', 'Info');", + "INSERT INTO audit_log (logged_at, kind, category, event, severity) VALUES (now() - @d * interval '1 day', @k, @c, 'x', 'Info');", new { d = daysOld, k = kind, c = category }); await Seed("audit", "SmtpAuth", 10); // security, older than 7d → purged diff --git a/tests/Dispatch.Data.Tests/SqlRepositoriesTests.cs b/tests/Dispatch.Data.Tests/SqlRepositoriesTests.cs index 85d38934..b5d22e1b 100644 --- a/tests/Dispatch.Data.Tests/SqlRepositoriesTests.cs +++ b/tests/Dispatch.Data.Tests/SqlRepositoriesTests.cs @@ -6,8 +6,8 @@ namespace Dispatch.Data.Tests; /// -/// Integration tests against a real SQL Server (Azure SQL Edge). Auto-skip when DISPATCH_TEST_SQL is unset. -/// Run with: DISPATCH_TEST_SQL="Server=localhost,1433;User Id=sa;Password=...;TrustServerCertificate=True;Encrypt=False" dotnet test +/// Integration tests against a real PostgreSQL server. Auto-skip when DISPATCH_TEST_SQL is unset. +/// Run with: DISPATCH_TEST_SQL="Host=localhost;Port=5432;Username=postgres;Password=..." dotnet test /// public class SqlRepositoriesTests(SqlServerFixture sql) : IClassFixture { @@ -33,7 +33,7 @@ public async Task Initializer_creates_schema_without_unconfigured_placeholder() // Migration 0003 removed the seeded "Unconfigured" placeholder relay - the first-run wizard creates // the first real relay (which becomes the catch-all). No placeholder should remain. var placeholders = await cn.ExecuteScalarAsync( - "SELECT COUNT(*) FROM relays WHERE provider = 'Unconfigured' AND is_default = 1"); + "SELECT COUNT(*) FROM relays WHERE provider = 'Unconfigured' AND is_default"); Assert.Equal(0, placeholders); } @@ -102,12 +102,12 @@ public async Task Log_retention_purge_deletes_aged_rows() var maintenance = new SqlLogMaintenance(sql.Factory); var spoolId = Guid.NewGuid().ToString("N"); - // Insert a Delivered row dated 100 days ago (bypassing the SYSUTCDATETIME default). + // Insert a Delivered row dated 100 days ago (bypassing the now() default). await using (var cn = await sql.Factory.OpenAsync()) { await cn.ExecuteAsync(""" INSERT INTO relay_log (logged_at, spool_id, event, status, from_address, from_domain, to_addresses, to_domain, subject) - VALUES (DATEADD(DAY, -100, SYSUTCDATETIME()), @spoolId, 'Delivered', 'OK', 'a@x.com', 'x.com', '[]', 'y.com', 's'); + VALUES (now() - interval '100 days', @spoolId, 'Delivered', 'OK', 'a@x.com', 'x.com', '[]', 'y.com', 's'); """, new { spoolId }); } @@ -483,15 +483,18 @@ await log.InsertAsync(new RelayLogEntry Assert.Equal(2, byRule.Rows.Count); Assert.All(byRule.Rows, r => Assert.Equal(spoolId, r.SpoolId)); - // Subject substring filter: matches case-insensitively; LIKE wildcards in the value are literal. + // Subject substring filter: case-sensitive (PostgreSQL LIKE); LIKE wildcards in the value are literal. await log.InsertAsync(new RelayLogEntry { Event = "Delivered", Status = "OK", SpoolId = Guid.NewGuid().ToString("N"), Subject = "Invoice #4242 ready", FromAddress = "a@x.com", FromDomain = "x.com", ToAddresses = ["b@" + domain], ToDomain = domain, RelayName = "alpha", }); - var bySubject = await query.QueryAsync(new MessageLogFilter { ToDomain = domain, Subject = "invoice" }); + var bySubject = await query.QueryAsync(new MessageLogFilter { ToDomain = domain, Subject = "Invoice" }); Assert.Single(bySubject.Rows); Assert.Contains("Invoice", bySubject.Rows[0].Subject); + // Case-sensitive: the wrong case does not match. + var wrongCase = await query.QueryAsync(new MessageLogFilter { ToDomain = domain, Subject = "invoice" }); + Assert.Empty(wrongCase.Rows); var noSubject = await query.QueryAsync(new MessageLogFilter { ToDomain = domain, Subject = "%nope%" }); Assert.Empty(noSubject.Rows); diff --git a/tests/Dispatch.Data.Tests/SqlServerFixture.cs b/tests/Dispatch.Data.Tests/SqlServerFixture.cs index 829c262c..eff72619 100644 --- a/tests/Dispatch.Data.Tests/SqlServerFixture.cs +++ b/tests/Dispatch.Data.Tests/SqlServerFixture.cs @@ -1,18 +1,18 @@ using Dapper; using Dispatch.Data; -using Microsoft.Data.SqlClient; +using Npgsql; using Microsoft.Extensions.Logging.Abstractions; namespace Dispatch.Data.Tests; /// -/// Spins up an isolated, uniquely-named database on the SQL server pointed to by the +/// Spins up an isolated, uniquely-named database on the PostgreSQL server pointed to by the /// DISPATCH_TEST_SQL environment variable, applies migrations, and drops it on dispose. /// When the env var is unset the fixture is = false and tests early-return, /// so the suite stays green without a database - UNLESS DISPATCH_REQUIRE_SQL is set (CI), in which /// case a missing connection string is a hard failure so the integration tests can never silently skip. -/// Before applying migrations it waits for the server to accept connections (SQL Edge can take tens of -/// seconds to become ready in CI), so a slow-starting container surfaces as a wait, not a flaky failure. +/// Before applying migrations it waits for the server to accept connections (a fresh container can take +/// tens of seconds to become ready in CI), so a slow-starting container surfaces as a wait, not a flaky failure. /// public sealed class SqlServerFixture : IAsyncLifetime { @@ -32,13 +32,14 @@ public async Task InitializeAsync() // every integration test and reporting a false green. if (IsTruthy(Environment.GetEnvironmentVariable("DISPATCH_REQUIRE_SQL"))) throw new InvalidOperationException( - "DISPATCH_REQUIRE_SQL is set but DISPATCH_TEST_SQL is not - the SQL integration tests cannot run."); + "DISPATCH_REQUIRE_SQL is set but DISPATCH_TEST_SQL is not - the PostgreSQL integration tests cannot run."); return; } _baseConnection = baseConn; - _dbName = "DispatchTest_" + Guid.NewGuid().ToString("N"); - ConnectionString = new SqlConnectionStringBuilder(baseConn) { InitialCatalog = _dbName }.ConnectionString; + // Postgres folds unquoted identifiers to lowercase; Guid's "N" format is already lowercase hex. + _dbName = "dispatchtest_" + Guid.NewGuid().ToString("N"); + ConnectionString = new NpgsqlConnectionStringBuilder(baseConn) { Database = _dbName }.ConnectionString; await WaitForServerAsync(TimeSpan.FromSeconds(90)); @@ -48,14 +49,14 @@ public async Task InitializeAsync() private async Task WaitForServerAsync(TimeSpan timeout) { - var master = new SqlConnectionStringBuilder(_baseConnection!) { InitialCatalog = "master", ConnectTimeout = 5 }.ConnectionString; + var maintenance = new NpgsqlConnectionStringBuilder(_baseConnection!) { Database = "postgres", Timeout = 5 }.ConnectionString; var deadline = DateTime.UtcNow + timeout; Exception? last = null; while (DateTime.UtcNow < deadline) { try { - await using var cn = new SqlConnection(master); + await using var cn = new NpgsqlConnection(maintenance); await cn.OpenAsync(); await cn.ExecuteAsync("SELECT 1"); return; @@ -67,7 +68,7 @@ private async Task WaitForServerAsync(TimeSpan timeout) } } throw new InvalidOperationException( - $"SQL server at the DISPATCH_TEST_SQL endpoint was not reachable within {timeout.TotalSeconds:F0}s.", last); + $"PostgreSQL server at the DISPATCH_TEST_SQL endpoint was not reachable within {timeout.TotalSeconds:F0}s.", last); } private static bool IsTruthy(string? v) => @@ -78,10 +79,10 @@ public async Task DisposeAsync() { if (_baseConnection is null || _dbName is null) return; - var master = new SqlConnectionStringBuilder(_baseConnection) { InitialCatalog = "master" }.ConnectionString; - await using var cn = new SqlConnection(master); + var maintenance = new NpgsqlConnectionStringBuilder(_baseConnection) { Database = "postgres" }.ConnectionString; + await using var cn = new NpgsqlConnection(maintenance); await cn.OpenAsync(); - await cn.ExecuteAsync( - $"ALTER DATABASE [{_dbName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_dbName}];"); + // FORCE (PG 13+) terminates any lingering backends so the drop can't hang on a stray connection. + await cn.ExecuteAsync($"DROP DATABASE IF EXISTS \"{_dbName}\" WITH (FORCE);"); } } diff --git a/tests/Dispatch.Data.Tests/SqlSettingsProviderTests.cs b/tests/Dispatch.Data.Tests/SqlSettingsProviderTests.cs index 6fc25b16..9b92b400 100644 --- a/tests/Dispatch.Data.Tests/SqlSettingsProviderTests.cs +++ b/tests/Dispatch.Data.Tests/SqlSettingsProviderTests.cs @@ -6,7 +6,7 @@ namespace Dispatch.Data.Tests; /// /// Unit tests for the cached, SQL-config-backed settings providers (spec §12.3, §12.5). These exercise -/// the override/fallback logic against an in-memory config store, so they need no SQL Server. +/// the override/fallback logic against an in-memory config store, so they need no database. /// public class SqlSettingsProviderTests { diff --git a/tests/Dispatch.Web.Tests/WebTestHost.cs b/tests/Dispatch.Web.Tests/WebTestHost.cs index c3453683..422c6cae 100644 --- a/tests/Dispatch.Web.Tests/WebTestHost.cs +++ b/tests/Dispatch.Web.Tests/WebTestHost.cs @@ -255,10 +255,10 @@ internal sealed class FakeLogMaintenance : ILogMaintenance { // 142 MB so /health's dbSizeMb is asserted as 142 in the web test. public Task GetDatabaseSizeBytesAsync(CancellationToken ct = default) => Task.FromResult(142L * 1024 * 1024); - public Task GetDatabaseUsedBytesAsync(CancellationToken ct = default) => Task.FromResult(100L * 1024 * 1024); - public Task IsSizeCappedEditionAsync(CancellationToken ct = default) => Task.FromResult(true); + public Task<(long TableBytes, long RowCount)> GetRelayLogStatsAsync(CancellationToken ct = default) => Task.FromResult((100L * 1024 * 1024, 1000L)); public Task PurgeByRetentionAsync(string @event, int retentionDays, CancellationToken ct = default) => Task.FromResult(0); public Task ArchiveAndDeleteOldestRelayLogAsync(int batch, Dispatch.Core.Maintenance.ArchiveRows archive, CancellationToken ct = default) => Task.FromResult(0); + public Task VacuumLogTablesAsync(CancellationToken ct = default) => Task.CompletedTask; } internal sealed class FakeStorageReport : Dispatch.Core.Maintenance.IStorageReport