diff --git a/.github/workflows/build-pg-focal.yml b/.github/workflows/build-pg-focal.yml new file mode 100644 index 00000000..10676a2f --- /dev/null +++ b/.github/workflows/build-pg-focal.yml @@ -0,0 +1,262 @@ +name: Build PostgreSQL core (focal) + +# Builds PostgreSQL *core* .deb packages for Ubuntu 20.04 (focal) by rebuilding +# the official Debian source package (PGDG dropped focal binaries upstream), then +# signs them with debsigs (--sign=maint) using the existing packaging key. +# +# One parameterized pipeline covers every focal-buildable major (PG 13..17). The +# default "all" matrix builds 13..16; PG17 is excluded up front and is built on +# demand by dispatching pg_major=17. workflow_dispatch can also target a single +# major and pin its minor / orig sha256. +# +# This is intentionally a *standalone* pipeline: it does not use the extension +# build flow (citus_package / pg_buildext / the build-package.yml matrix), which +# assumes PostgreSQL itself comes from PGDG. + +on: + workflow_dispatch: + inputs: + pg_major: + description: "PostgreSQL major to build (\"all\" builds 13-16; pick 17 explicitly if needed)" + required: true + type: choice + default: "all" + options: + - "all" + - "13" + - "14" + - "15" + - "16" + - "17" + pg_upstream_version: + description: "Optional: pin the minor (e.g. 16.15). Blank = latest. Only used when a single major is selected." + required: false + default: "" + pg_orig_sha256: + description: "Optional: pin sha256 of the orig.tar.bz2 (blank = auto-resolve from the official .dsc). Only used when a single major is selected." + required: false + default: "" + run_tests: + description: "Run upstream regression suite (1=yes, slower)" + required: false + default: "0" + push: + branches: + - pg-focal + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + setup: + name: Resolve build matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Compute pg_major matrix + id: matrix + run: | + sel="${{ github.event.inputs.pg_major }}" + if [ -z "${sel}" ] || [ "${sel}" = "all" ]; then + # PG17 is intentionally excluded from "all" -- not needed up front. + # It remains fully supported: dispatch with pg_major=17 to build it. + echo 'matrix={"pg_major":["13","14","15","16"]}' >> "$GITHUB_OUTPUT" + else + echo "matrix={\"pg_major\":[\"${sel}\"]}" >> "$GITHUB_OUTPUT" + fi + + build-and-sign: + needs: setup + name: Build & sign PG${{ matrix.pg_major }} (focal) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} + env: + PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} + PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} + PG_MAJOR: ${{ matrix.pg_major }} + # Minor / sha pins only make sense for a single explicitly-selected major; + # ignore them on the "all" matrix so each major still auto-resolves latest. + PG_UPSTREAM_VERSION: ${{ github.event.inputs.pg_major != 'all' && github.event.inputs.pg_upstream_version || '' }} + PG_ORIG_SHA256: ${{ github.event.inputs.pg_major != 'all' && github.event.inputs.pg_orig_sha256 || '' }} + RUN_TESTS: ${{ github.event.inputs.run_tests || '0' }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USER_NAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build focal builder image + run: | + docker build -t focal-pg-builder \ + -f dockerfiles/focal-pg-builder/Dockerfile . + + - name: Build PostgreSQL ${{ matrix.pg_major }} packages + run: | + mkdir -p packages + docker run --rm \ + -e PG_MAJOR="${PG_MAJOR}" \ + -e PG_UPSTREAM_VERSION="${PG_UPSTREAM_VERSION}" \ + -e PG_ORIG_SHA256="${PG_ORIG_SHA256}" \ + -e RUN_TESTS="${RUN_TESTS}" \ + -v "${PWD}/packages:/packages" \ + focal-pg-builder + echo "Built packages:" + ls -1 "packages/focal/pg${PG_MAJOR}"/*.deb + + - name: Sign packages (debsigs --sign=maint) + # Use the prebuilt, deployed debsigner image (the one all Citus signing + # uses), not a locally built copy of dockerfiles/debsigner, which has + # drifted from it. Mirrors tools.packaging_automation.citus_package. + # + # The signer's entrypoint signs exactly "/packages/*/*.deb" (one dir + # level deep), so mount the parent of the per-major output dir: with + # "${PWD}/packages/focal:/packages" the debs land at /packages/pg/*.deb, + # which is what that glob expects. + run: | + if [ -z "${PACKAGING_SECRET_KEY}" ] || [ -z "${PACKAGING_PASSPHRASE}" ]; then + echo "::error::PACKAGING_SECRET_KEY / PACKAGING_PASSPHRASE secrets are not set" >&2 + exit 1 + fi + printf '%s' "${PACKAGING_PASSPHRASE}" | docker run --rm -i \ + -e PACKAGING_SECRET_KEY \ + -e PACKAGING_PASSPHRASE \ + -v "${PWD}/packages/focal:/packages" \ + citusdata/packaging:debsigner + + - name: Verify signatures are embedded + run: | + rc=0 + for deb in "packages/focal/pg${PG_MAJOR}"/*.deb; do + if ar t "$deb" | grep -q '^_gpgmaint$'; then + echo "signed: $deb" + else + echo "::error::missing _gpgmaint signature in $deb" >&2 + rc=1 + fi + done + exit $rc + + - name: Upload signed packages + uses: actions/upload-artifact@v4 + with: + name: postgresql-${{ matrix.pg_major }}-focal-deb + path: | + packages/focal/pg${{ matrix.pg_major }}/*.deb + packages/focal/pg${{ matrix.pg_major }}/*.changes + packages/focal/pg${{ matrix.pg_major }}/*.buildinfo + if-no-files-found: error + + assemble: + needs: build-and-sign + name: Assemble combined co-installable set + runs-on: ubuntu-latest + steps: + - name: Download all per-major package sets + uses: actions/download-artifact@v4 + with: + pattern: postgresql-*-focal-deb + path: per-major # -> per-major/postgresql--focal-deb/*.deb + + - name: Assemble de-duplicated set + # PostgreSQL builds six shared, single-instance system libraries from + # *every* major's source -- libpq5, libpq-dev, libpgtypes3, libecpg6, + # libecpg-dev, libecpg-compat3 (plus their -dbgsym). They share one + # package name but carry a per-major version, so they cannot be + # co-installed; the newest copy satisfies every major's ">=" dependency. + # Keep all per-major packages from every major, but keep the shared + # libraries only from the highest major present. The result installs + # every built major side by side with no conflicts. Packages stay + # byte-identical (already signed) -- we only copy, never repackage. + run: | + set -euo pipefail + shared_re='^(libpq5|libpq-dev|libpgtypes3|libecpg6|libecpg-dev|libecpg-compat3)(-dbgsym)?$' + + # Discover the majors we actually received and pick the highest. + majors="$(find per-major -maxdepth 1 -type d -name 'postgresql-*-focal-deb' \ + | sed -E 's#.*/postgresql-([0-9]+)-focal-deb#\1#' | sort -n)" + [ -n "${majors}" ] || { echo "::error::no per-major artifacts found" >&2; exit 1; } + newest="$(echo "${majors}" | tail -1)" + echo "majors present: $(echo ${majors} | tr '\n' ' '); shared libs taken from PG${newest}" + + mkdir -p postgresql-all-focal + for m in ${majors}; do + for deb in per-major/postgresql-${m}-focal-deb/*.deb; do + [ -e "${deb}" ] || continue + pkg="$(dpkg-deb -f "${deb}" Package)" + if [[ "${pkg}" =~ ${shared_re} ]] && [ "${m}" != "${newest}" ]; then + echo " skip shared ${pkg} from PG${m} (kept from PG${newest})" + continue + fi + cp "${deb}" postgresql-all-focal/ + done + done + + echo "==> Assembled $(ls postgresql-all-focal/*.deb | wc -l) packages" + echo "==> Shared libraries in the combined set (must be exactly one version each):" + for deb in postgresql-all-focal/*.deb; do + pkg="$(dpkg-deb -f "${deb}" Package)" + if [[ "${pkg}" =~ ${shared_re} ]]; then + dpkg-deb -f "${deb}" Package Version | tr '\n' ' '; echo + fi + done | sort -u + + - name: Verify the combined set has no duplicate package names + run: | + set -euo pipefail + dupes="$(for deb in postgresql-all-focal/*.deb; do dpkg-deb -f "${deb}" Package; done \ + | sort | uniq -d)" + if [ -n "${dupes}" ]; then + echo "::error::duplicate package names in combined set (would conflict on install):" >&2 + echo "${dupes}" >&2 + exit 1 + fi + echo "OK: every package name appears exactly once" + + - name: Upload combined co-installable set + uses: actions/upload-artifact@v4 + with: + name: postgresql-all-focal + path: postgresql-all-focal/*.deb + if-no-files-found: error + + install-smoke-test: + needs: assemble + name: Install smoke test (focal) + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download combined co-installable set + uses: actions/download-artifact@v4 + with: + name: postgresql-all-focal + path: debs + + - name: Install and verify in a clean focal container + # The jobs above only prove the packages exist, are signed, and have + # unique names -- not that they can actually be installed. This installs + # the whole shipped set (including -dbgsym) into a stock ubuntu:20.04, + # starts every cluster, and checks JIT is live. + # + # See scripts/smoke_test_focal_debs for why the PGDG archive's "main" + # component has to be enabled: focal ships postgresql-common + # 214ubuntu0.1, but the server packages need >= 252~. + run: | + docker run --rm \ + -v "${PWD}/debs:/debs:ro" \ + -v "${PWD}/scripts/smoke_test_focal_debs:/usr/local/bin/smoke_test_focal_debs:ro" \ + -e DEBS_DIR=/debs \ + ubuntu:20.04 \ + /usr/local/bin/smoke_test_focal_debs diff --git a/dockerfiles/focal-pg-builder/Dockerfile b/dockerfiles/focal-pg-builder/Dockerfile new file mode 100644 index 00000000..1008474e --- /dev/null +++ b/dockerfiles/focal-pg-builder/Dockerfile @@ -0,0 +1,67 @@ +# vim:set ft=dockerfile: +# +# Generic builder image for PostgreSQL *core* packages targeting Ubuntu 20.04 +# (focal). One image serves every focal-buildable major (PG 13, 14, 15, 16, 17); +# the major is selected at run time via PG_MAJOR (see scripts/build_pg_focal). +# +# Why this exists: +# apt.postgresql.org (PGDG) no longer ships PostgreSQL binaries for focal +# (focal reached EOL standard support 2025-04). To get a newer minor on focal +# we rebuild the official Debian source package ourselves. +# +# Strategy (validated): +# - Upstream tarball: postgresql-_.orig.tar.bz2 (from the live pool) +# - Debian packaging: the frozen focal-era debian/ (the last +# ".-N.pgdg20.04+1" PGDG built for focal), because its build profile +# uses focal's *default* toolchain (clang/llvm-dev = LLVM 10), so the +# resulting JIT depends on focal's libllvm10 (installable on focal), unlike +# newer packaging which requires clang-19/llvm-19. +# - Build tooling (debhelper 13, dh-exec, postgresql-common-dev) is restored +# from the PGDG *archive* (apt-archive.postgresql.org), which keeps the +# removed focal-pgdg suite. +# +# The heavy lifting lives in scripts/build_pg_focal (the entrypoint). +FROM ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + +# PGDG repository signing key fingerprint: +# B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl gnupg; \ + install -d /usr/share/keyrings; \ + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg; \ + # The focal-pgdg suite exposes a 'main' component (build tooling) plus a + # per-major component for each PostgreSQL major. List every focal-buildable + # major so the same image works for PG 13..17. + echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 13 14 15 16 17" \ + > /etc/apt/sources.list.d/pgdg-archive.list; \ + # make sure 'universe' is enabled (clang / llvm-dev live there on focal) + sed -i 's/^# deb \(.*universe\)/deb \1/' /etc/apt/sources.list; \ + apt-get update; \ + # base build tooling; the per-build Build-Depends are resolved at run time + # by scripts/build_pg_focal via mk-build-deps against debian/control. + apt-get install -y --no-install-recommends \ + build-essential \ + devscripts \ + equivs \ + fakeroot \ + quilt \ + dpkg-dev \ + debhelper \ + dh-exec \ + postgresql-common-dev \ + xz-utils \ + bzip2; \ + rm -rf /var/lib/apt/lists/* + +# Fail the image build early if the archived focal-pgdg debhelper (>= 13) is not +# what we picked up (debhelper-compat (= 13) is required by the packaging). +RUN dpkg-query -W -f='${Package} ${Version}\n' debhelper postgresql-common-dev dh-exec + +COPY scripts/build_pg_focal /usr/local/bin/build_pg_focal +RUN chmod +x /usr/local/bin/build_pg_focal + +VOLUME /packages +ENTRYPOINT ["/usr/local/bin/build_pg_focal"] diff --git a/scripts/build_pg_focal b/scripts/build_pg_focal new file mode 100755 index 00000000..ae29208d --- /dev/null +++ b/scripts/build_pg_focal @@ -0,0 +1,227 @@ +#!/bin/bash +# +# build_pg_focal -- rebuild PostgreSQL *core* Debian packages for Ubuntu 20.04 (focal). +# +# PGDG stopped shipping focal binaries once focal reached EOL standard support, so +# the only way to get newer minors on focal is to rebuild the official Debian +# source package ourselves. This works for every major that ever had a focal +# build *and* still receives new upstream minors: PG 13, 14, 15, 16, 17. +# +# Approach (see dockerfiles/focal-pg-builder/Dockerfile for the rationale): +# newer upstream orig.tar.bz2 + the frozen focal-era debian/ packaging +# (the last ".-N.pgdg20.04+1" PGDG published for focal) +# -> dpkg-buildpackage inside a focal environment that has the archived +# focal-pgdg build tooling available. +# +# The focal-era debian/ packaging matters: its *default* build profile uses +# focal's default toolchain (clang/llvm-dev = LLVM 10), so the resulting JIT +# depends on focal's libllvm10 (installable on focal) instead of the +# clang-19/llvm-19 that newer packaging requires. +# +# Output: unsigned *.deb in ${OUTPUT_DIR} (default /packages/focal/pg). +# Signing is a separate step performed by the debsigner image (debsigs +# --sign=maint), so the build and the signing key never live in the same +# container. +# +# All inputs are overridable via environment variables. The common cases need +# nothing but PG_MAJOR: +# * PG_UPSTREAM_VERSION defaults to the latest minor PGDG publishes for the +# major, so a new minor release needs no edit here. +# * PG_DEBIAN_BASE defaults to the known last focal packaging for the major +# (focal is EOL, so these are frozen); override only if PGDG ever re-touches +# the archive. +# +# Examples: +# PG_MAJOR=15 build_pg_focal # latest 15.x +# PG_MAJOR=16 PG_UPSTREAM_VERSION=16.15 build_pg_focal +# PG_MAJOR=17 PG_DROP_PATCHES="hurd-iovec foo" build_pg_focal + +set -euo pipefail + +# ---------------------------------------------------------------------------- +# Inputs (override via env) +# ---------------------------------------------------------------------------- +PG_MAJOR="${PG_MAJOR:?ERROR: set PG_MAJOR (one of: 13 14 15 16 17)}" + +case "${PG_MAJOR}" in + 13|14|15|16|17) : ;; + *) echo "ERROR: PG_MAJOR='${PG_MAJOR}' is not a focal-buildable major (expected 13|14|15|16|17)." >&2 + echo " (11/12 are EOL with no minor newer than their last focal build; 18 never shipped on focal.)" >&2 + exit 64 ;; +esac + +# Sources: orig tarball from the live pool (shared across distros), focal-era +# debian/ packaging from the archive (removed from the live pool with focal). +LIVE_POOL="https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-${PG_MAJOR}" +ARCHIVE_POOL="https://apt-archive.postgresql.org/pub/repos/apt/pool/main/p/postgresql-${PG_MAJOR}" + +# default_debian_base -- the last focal packaging PGDG published per major. +# These are frozen (focal is EOL); override with PG_DEBIAN_BASE if ever needed. +default_debian_base() { + case "${PG_MAJOR}" in + 13) echo "13.21-1.pgdg20.04+1" ;; + 14) echo "14.18-1.pgdg20.04+1" ;; + 15) echo "15.13-1.pgdg20.04+1" ;; + 16) echo "16.9-1.pgdg20.04+1" ;; + 17) echo "17.5-1.pgdg20.04+1" ;; + esac +} + +# find_latest_upstream_minor -- newest . orig tarball in the live pool. +# The orig tarball is shared across every per-distro revision, so the live pool +# still carries it long after focal binaries were dropped. +find_latest_upstream_minor() { + curl -fsSL "${LIVE_POOL}/" \ + | grep -oE "postgresql-${PG_MAJOR}_${PG_MAJOR}\.[0-9]+\.orig\.tar\.bz2" \ + | sed -E "s/.*_(${PG_MAJOR}\.[0-9]+)\.orig.*/\1/" \ + | sort -V | tail -1 +} + +UPSTREAM="${PG_UPSTREAM_VERSION:-}" # upstream PostgreSQL minor to build +if [ -z "${UPSTREAM}" ]; then + UPSTREAM="$(find_latest_upstream_minor || true)" + [ -n "${UPSTREAM}" ] || { echo "ERROR: could not auto-resolve the latest ${PG_MAJOR}.x minor from ${LIVE_POOL}." >&2; exit 1; } + echo "==> PG_UPSTREAM_VERSION not set; auto-resolved latest minor: ${UPSTREAM}" +fi + +DEBIAN_BASE="${PG_DEBIAN_BASE:-$(default_debian_base)}" # focal-era packaging to reuse +TARGET_VERSION="${PG_TARGET_VERSION:-${UPSTREAM}-1.pgdg20.04+1}" # produced package version + +# sha256 of the artifacts we download. Leave empty to auto-resolve from the +# official .dsc; set to pin a specific value (integrity / reproducibility). +ORIG_SHA256="${PG_ORIG_SHA256:-}" +DEBIAN_SHA256="${PG_DEBIAN_SHA256:-}" + +# Patches in the frozen focal debian/ packaging that no longer apply to newer +# upstream (e.g. merged upstream). Space-separated; each is dropped from the +# quilt series only if present, so the default is safe for every major: +# * hurd-iovec was merged upstream as of 16.14 (pg_iovec.h guards IOV_MAX), +# so it fails to apply on 16.x/17.x but is absent from 13/14/15 packaging. +# If a future minor introduces new drift, append the offending patch name here +# (or via the PG_DROP_PATCHES env) -- no other code change required. +PG_DROP_PATCHES="${PG_DROP_PATCHES:-hurd-iovec}" + +# Set RUN_TESTS=1 to run the upstream regression suite (slower, needs more deps). +RUN_TESTS="${RUN_TESTS:-0}" + +OUTPUT_DIR="${OUTPUT_DIR:-/packages/focal/pg${PG_MAJOR}}" + +ORIG_TARBALL="postgresql-${PG_MAJOR}_${UPSTREAM}.orig.tar.bz2" +DEBIAN_TARBALL="postgresql-${PG_MAJOR}_${DEBIAN_BASE}.debian.tar.xz" + +WORK="${WORK_DIR:-/build}" +SRCDIR="${WORK}/postgresql-${PG_MAJOR}-${UPSTREAM}" + +# changelog identity (only cosmetic for a binary build) +export DEBEMAIL="${DEBEMAIL:-pgsql-pkg-debian@lists.postgresql.org}" +export DEBFULLNAME="${DEBFULLNAME:-PostgreSQL focal rebuild}" + +# dsc_sha256 -> prints the sha256 recorded for +# in the Checksums-Sha256 stanza of a Debian .dsc (served over HTTPS by PGDG). +dsc_sha256() { + curl -fsSL "$1" | awk -v f="$2" ' + /^Checksums-Sha256:/ { insec=1; next } + /^[A-Za-z]/ { insec=0 } + insec && $3==f { print $1; exit }' +} + +# Find any released .dsc for this upstream version (the orig tarball is shared +# across every per-distro revision, so any of them carries its checksum). +find_orig_dsc_url() { + local v="${UPSTREAM//./\\.}" + curl -fsSL "${LIVE_POOL}/" \ + | grep -oE "postgresql-${PG_MAJOR}_${v}-[0-9]+\.pgdg[A-Za-z0-9.+~]*\.dsc" \ + | sort -u | tail -1 +} + +echo "==> Building PostgreSQL ${UPSTREAM} (PG${PG_MAJOR}) as ${TARGET_VERSION} (focal)" +echo " debian/ packaging base: ${DEBIAN_BASE}" +mkdir -p "${WORK}" "${OUTPUT_DIR}" +cd "${WORK}" + +echo "==> [1/6] Download source artifacts" +curl -fSL "${LIVE_POOL}/${ORIG_TARBALL}" -o "${ORIG_TARBALL}" +curl -fSL "${ARCHIVE_POOL}/${DEBIAN_TARBALL}" -o "${DEBIAN_TARBALL}" + +echo "==> [2/6] Resolve & verify checksums" +if [ -z "${ORIG_SHA256}" ]; then + orig_dsc="$(find_orig_dsc_url || true)" + [ -n "${orig_dsc}" ] || { echo "ERROR: could not find a .dsc for ${ORIG_TARBALL} (is ${UPSTREAM} released?)." >&2; exit 1; } + ORIG_SHA256="$(dsc_sha256 "${LIVE_POOL}/${orig_dsc}" "${ORIG_TARBALL}")" + [ -n "${ORIG_SHA256}" ] || { echo "ERROR: ${ORIG_TARBALL} not listed in ${orig_dsc}." >&2; exit 1; } + echo " orig sha256 resolved from ${orig_dsc}: ${ORIG_SHA256}" +else + echo " orig sha256 pinned: ${ORIG_SHA256}" +fi +if [ -z "${DEBIAN_SHA256}" ]; then + DEBIAN_SHA256="$(dsc_sha256 "${ARCHIVE_POOL}/postgresql-${PG_MAJOR}_${DEBIAN_BASE}.dsc" "${DEBIAN_TARBALL}")" + [ -n "${DEBIAN_SHA256}" ] || { echo "ERROR: ${DEBIAN_TARBALL} not listed in postgresql-${PG_MAJOR}_${DEBIAN_BASE}.dsc." >&2; exit 1; } + echo " debian sha256 resolved from postgresql-${PG_MAJOR}_${DEBIAN_BASE}.dsc: ${DEBIAN_SHA256}" +else + echo " debian sha256 pinned: ${DEBIAN_SHA256}" +fi +echo "${ORIG_SHA256} ${ORIG_TARBALL}" | sha256sum -c - +echo "${DEBIAN_SHA256} ${DEBIAN_TARBALL}" | sha256sum -c - + +echo "==> [3/6] Assemble 3.0 (quilt) source tree" +rm -rf "${SRCDIR}" +mkdir -p "${SRCDIR}" +tar -xf "${ORIG_TARBALL}" -C "${SRCDIR}" --strip-components=1 +tar -xf "${DEBIAN_TARBALL}" -C "${SRCDIR}" # unpacks debian/ +# NB: ${ORIG_TARBALL} is already named postgresql-${PG_MAJOR}_${UPSTREAM}.orig.tar.bz2, +# exactly what dpkg-buildpackage expects one level above ${SRCDIR}. + +cd "${SRCDIR}" + +# Drop patches from the frozen focal packaging that no longer apply to the newer +# upstream (each removed only if present, so this is a no-op where unneeded). +for p in ${PG_DROP_PATCHES}; do + if grep -qx "${p}" debian/patches/series 2>/dev/null; then + sed -i "/^${p}$/d" debian/patches/series + rm -f "debian/patches/${p}" + echo " dropped patch (PG_DROP_PATCHES): ${p}" + fi +done + +echo "==> [4/6] Verify the quilt patch series applies cleanly to ${UPSTREAM}" +export QUILT_PATCHES=debian/patches +if [ -s debian/patches/series ]; then + quilt push -a + quilt pop -a +fi + +echo "==> [5/6] Set version to ${TARGET_VERSION} and install Build-Depends" +dch --newversion "${TARGET_VERSION}" --distribution focal --force-distribution \ + "Rebuild of PostgreSQL ${UPSTREAM} for focal (upstream PGDG focal-pgdg discontinued)." + +BUILD_PROFILES="" +BUILD_OPTIONS="parallel=$(nproc)" +if [ "${RUN_TESTS}" != "1" ]; then + BUILD_PROFILES="nocheck" + BUILD_OPTIONS="${BUILD_OPTIONS} nocheck" +fi +export DEB_BUILD_PROFILES="${BUILD_PROFILES}" + +apt-get update +mk-build-deps --install --remove \ + --tool 'apt-get -o Debug::pkgProblemResolver=yes --yes --no-install-recommends' \ + debian/control + +echo "==> [6/6] Build binary packages (DEB_BUILD_OPTIONS='${BUILD_OPTIONS}')" +export DEB_BUILD_OPTIONS="${BUILD_OPTIONS}" +dpkg-buildpackage -b -uc -us + +echo "==> Collect artifacts into ${OUTPUT_DIR}" +cp -v "${WORK}"/*.deb "${OUTPUT_DIR}/" +# Debug-symbol packages are emitted as .ddeb on Ubuntu (e.g. postgresql-16-dbgsym +# _..._amd64.ddeb). Ship them as .deb (the Debian convention; the on-disk format +# is identical) so they flow through the same signing/verification/publishing as +# every other package instead of being silently dropped by the *.deb glob above. +for ddeb in "${WORK}"/*.ddeb; do + [ -e "${ddeb}" ] || continue + cp -v "${ddeb}" "${OUTPUT_DIR}/$(basename "${ddeb}" .ddeb).deb" +done +cp -v "${WORK}"/*.buildinfo "${WORK}"/*.changes "${OUTPUT_DIR}/" 2>/dev/null || true + +echo "==> DONE. Packages:" +ls -1 "${OUTPUT_DIR}"/*.deb diff --git a/scripts/smoke_test_focal_debs b/scripts/smoke_test_focal_debs new file mode 100755 index 00000000..d9d2d4ab --- /dev/null +++ b/scripts/smoke_test_focal_debs @@ -0,0 +1,182 @@ +#!/bin/bash +# +# smoke_test_focal_debs -- install the built PostgreSQL core .deb set inside a +# clean Ubuntu 20.04 (focal) container and prove it actually works. +# +# The build pipeline already checks that the packages *exist*, are signed, and +# carry unique package names. None of that proves they are installable: the +# combined set deliberately ships the six single-instance shared libraries +# (libpq5, libpq-dev, libpgtypes3, libecpg6, libecpg-dev, libecpg-compat3) from +# only the highest major, on the assumption that e.g. libpq5 16.x satisfies +# PG13's "libpq5 (>= 13~~)". This script tests that assumption for real. +# +# It also asserts JIT is live (SELECT pg_jit_available()). PostgreSQL here links +# libllvm10 directly -- there is no separate postgresql-NN-jit-llvm package -- +# so a regression in the frozen-focal-packaging / LLVM-10 strategy that silently +# disables JIT would otherwise ship unnoticed. +# +# NOTE ON CLUSTER CREATION +# ------------------------ +# Installing several majors at once produces exactly *one* cluster, not one per +# major: postgresql-common's create_main_cluster() returns early if any cluster +# already exists, and again if the package being configured is not the newest +# installed major. That is intended upstream behaviour, so this script creates +# the remaining clusters explicitly rather than expecting them to appear. +# +# NOTE ON THE PGDG ARCHIVE REPOSITORY +# ----------------------------------- +# This is *not* a self-containment test. The produced server packages depend on +# postgresql-common (>= 229~ / 241~ / 252~ depending on major), but Ubuntu focal +# only ships 214ubuntu0.1. The only focal build that satisfies them is +# 278.pgdg20.04+1 from apt-archive.postgresql.org, so the archive's "main" +# component is enabled below. Consumers of these packages need it too -- that is +# the point of encoding it here rather than leaving it as tribal knowledge. +# +# Only "main" is enabled, never the per-major components: those still carry +# PGDG's own libpq5/libecpg6/... for focal and would shadow the packages under +# test. +# +# Usage (inside a stock ubuntu:20.04 container, as root): +# DEBS_DIR=/debs smoke_test_focal_debs +# +# Everything is overridable via environment variables: +# DEBS_DIR directory holding the *.deb set (default /debs) +# EXPECTED_PGCOMMON postgresql-common version to expect (default 278.pgdg20.04+1) +# PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) + +set -euo pipefail + +DEBS_DIR="${DEBS_DIR:-/debs}" +EXPECTED_PGCOMMON="${EXPECTED_PGCOMMON:-278.pgdg20.04+1}" +PGDG_ARCHIVE_SUITE="${PGDG_ARCHIVE_SUITE:-focal-pgdg}" +PGDG_ARCHIVE_URL="${PGDG_ARCHIVE_URL:-https://apt-archive.postgresql.org/pub/repos/apt}" + +export DEBIAN_FRONTEND=noninteractive +# Pin the locale so initdb is deterministic. C.UTF-8 always exists on focal and +# needs no locale-gen. +export LANG=C.UTF-8 + +fail=0 +note() { echo "==> $*"; } +err() { echo "::error::$*" >&2; fail=1; } + +shopt -s nullglob +debs=("${DEBS_DIR}"/*.deb) +shopt -u nullglob +[ ${#debs[@]} -gt 0 ] || { echo "ERROR: no .deb files found in ${DEBS_DIR}" >&2; exit 1; } + +note "[1/8] Preparing container environment (${#debs[@]} packages to install)" +# postgresql-NN.postinst ends in `invoke-rc.d postgresql start $VERSION` under +# `set -e`, and there is no systemd in a container. A policy-rc.d denying the +# action makes invoke-rc.d return 0, so the postinst still creates the cluster; +# we start the clusters explicitly further down instead. +printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d +chmod +x /usr/sbin/policy-rc.d + +apt-get update -qq +apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg >/dev/null + +note "[2/8] Enabling the PGDG archive (main only) for postgresql-common" +# PGDG repository signing key fingerprint: +# B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +install -d /usr/share/keyrings +curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg +echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} ${PGDG_ARCHIVE_SUITE} main" \ + > /etc/apt/sources.list.d/pgdg-archive.list +apt-get update -qq + +note "[3/8] Installing the full shipped set (including -dbgsym)" +apt-get install -y "${debs[@]}" + +note "[4/8] Verifying every shipped package installed at its exact version" +# Proves apt used *our* binaries and did not silently substitute a package of +# the same name from focal or from the PGDG archive. +for deb in "${debs[@]}"; do + pkg="$(dpkg-deb -f "${deb}" Package)" + want="$(dpkg-deb -f "${deb}" Version)" + got="$(dpkg-query -W -f='${Version}' "${pkg}" 2>/dev/null || echo MISSING)" + if [ "${got}" != "${want}" ]; then + err "${pkg}: installed ${got}, expected ${want}" + fi +done +if [ "${fail}" -eq 0 ]; then + note " all ${#debs[@]} packages installed at the expected versions" +fi + +note "[5/8] Verifying postgresql-common came from the archive at ${EXPECTED_PGCOMMON}" +pgc="$(dpkg-query -W -f='${Version}' postgresql-common 2>/dev/null || echo MISSING)" +if [ "${pgc}" != "${EXPECTED_PGCOMMON}" ]; then + err "postgresql-common is ${pgc}, expected ${EXPECTED_PGCOMMON}" +else + note " postgresql-common ${pgc}" +fi + +note "[6/8] Creating a cluster for every installed major" +# Derive the majors from the installed server binaries rather than hardcoding, +# so this works unchanged for a single-major dispatch as well as the full matrix. +mapfile -t majors < <(ls -v /usr/lib/postgresql/*/bin/postgres \ + | sed -E 's#.*/postgresql/([0-9]+)/bin/postgres#\1#') +[ ${#majors[@]} -gt 0 ] || { echo "ERROR: no PostgreSQL server binaries installed" >&2; exit 1; } +echo " installed majors: ${majors[*]}" + +# postgresql-common's create_main_cluster() deliberately creates at most ONE +# cluster when several majors are installed at once -- it returns early if any +# cluster already exists, and again if the package being configured is not the +# newest major. So exactly one "main" cluster (on the newest major) exists at +# this point. Create the rest explicitly: running every major side by side is +# precisely the property the assemble job's shared-library de-duplication +# assumes, and nothing else verifies it. +for m in "${majors[@]}"; do + if [ -e "/etc/postgresql/${m}/main/postgresql.conf" ]; then + echo " PG${m}: cluster already present (created by postinst)" + else + pg_createcluster -u postgres "${m}" main + fi +done + +note "[7/8] Starting every cluster" +for m in "${majors[@]}"; do + status="$(pg_lsclusters -h | awk -v v="${m}" '$1 == v && $2 == "main" { print $4 }')" + [ "${status}" = "online" ] || pg_ctlcluster "${m}" main start +done +pg_lsclusters + +note "[8/8] Querying each cluster (version + JIT availability)" +for m in "${majors[@]}"; do + port="$(pg_lsclusters -h | awk -v v="${m}" '$1 == v && $2 == "main" { print $3 }')" + status="$(pg_lsclusters -h | awk -v v="${m}" '$1 == v && $2 == "main" { print $4 }')" + if [ "${status}" != "online" ]; then + err "PG${m}: cluster is '${status}', expected 'online'" + continue + fi + ver="$(su postgres -c "psql -p ${port} -tAc 'SELECT version()'" | head -1)" + jit="$(su postgres -c "psql -p ${port} -tAc 'SELECT pg_jit_available()'" | tr -d '[:space:]')" + echo " PG${m} (port ${port}): ${ver}" + if [ "${jit}" != "t" ]; then + err "PG${m}: pg_jit_available() returned '${jit}', expected 't' (JIT is not available)" + else + echo " PG${m}: JIT available" + fi +done + +# Co-installability: every major that shipped a server package must be installed +# and running, each on its own port. +note "Verifying co-existence" +for m in $(printf '%s\n' "${debs[@]}" | sed -nE 's#.*/postgresql-([0-9]+)_.*#\1#p' | sort -un); do + if ! printf '%s\n' "${majors[@]}" | grep -qx "${m}"; then + err "PG${m} shipped a server package but is not installed" + fi +done +nports="$(pg_lsclusters -h | awk '{print $3}' | sort -u | wc -l)" +nclusters="$(pg_lsclusters -h | wc -l)" +if [ "${nports}" -ne "${nclusters}" ]; then + err "expected ${nclusters} distinct ports, found ${nports} (clusters are colliding)" +fi +echo " ${nclusters} clusters running on ${nports} distinct ports" + +if [ "${fail}" -ne 0 ]; then + echo "SMOKE TEST FAILED" >&2 + exit 1 +fi +echo "SMOKE TEST PASSED: ${#debs[@]} packages installed, ${nclusters} clusters running (PG ${majors[*]}), JIT available on all"