diff --git a/apps/workflow/tests/test_xero_instance_templates.py b/apps/workflow/tests/test_xero_instance_templates.py index 10ab3b92b..ee1f77857 100644 --- a/apps/workflow/tests/test_xero_instance_templates.py +++ b/apps/workflow/tests/test_xero_instance_templates.py @@ -14,6 +14,9 @@ ) INSTANCE_SCRIPT = REPO_ROOT / "scripts" / "server" / "instance.sh" DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "server" / "deploy.sh" +CUTOVER_LEGACY_SCRIPT = REPO_ROOT / "scripts" / "server" / "cutover_legacy_instance.sh" +LEGACY_ROLLBACK_SCRIPT = REPO_ROOT / "scripts" / "legacy_rollback.sh" +PREDEPLOY_BACKUP_SCRIPT = REPO_ROOT / "scripts" / "predeploy_backup.sh" COMMON_SCRIPT = REPO_ROOT / "scripts" / "server" / "common.sh" SERVER_SETUP_SCRIPT = REPO_ROOT / "scripts" / "server" / "server-setup.sh" SERVER_README = REPO_ROOT / "scripts" / "server" / "README.md" @@ -225,13 +228,13 @@ def test_release_utils_builds_immutable_release_artifacts(self) -> None: self.assertIn("RELEASES_DIR", content) self.assertIn("git -C '$LOCAL_REPO' archive '$sha'", content) - self.assertIn("printf '%s\\n' '$sha' > '$tmp_dir/.release-sha'", content) - self.assertIn("python3.12 -m venv '$tmp_dir/.venv'", content) + self.assertIn("printf '%s\\n' '$sha' > '$release_dir/.release-sha'", content) + self.assertIn("python3.12 -m venv '$release_dir/.venv'", content) self.assertIn("npm run check:typed-router", content) self.assertIn("npm run build", content) self.assertIn("npm run manual:build", content) self.assertIn("rm -rf node_modules", content) - self.assertIn("touch '$tmp_dir/.complete'", content) + self.assertIn("touch '$release_dir/.complete'", content) def test_runtime_templates_use_current_release(self) -> None: for template in [ @@ -390,7 +393,7 @@ def test_release_cleanup_is_deploy_integrated_and_reachability_based(self) -> No release_utils_content = RELEASE_UTILS.read_text() self.assertIn("--cleanup-releases", deploy_content) - self.assertIn("cleanup_stale_release_builds", deploy_content) + self.assertIn("cleanup_incomplete_releases", deploy_content) self.assertIn('cleanup_unreferenced_releases "$TARGET_SHA"', deploy_content) self.assertIn("release_is_referenced()", release_utils_content) self.assertIn( @@ -399,6 +402,130 @@ def test_release_cleanup_is_deploy_integrated_and_reachability_based(self) -> No ) self.assertIn("Removing unreferenced release", release_utils_content) + def test_legacy_cutover_rollback_artifacts_are_root_trusted(self) -> None: + cutover_content = CUTOVER_LEGACY_SCRIPT.read_text() + rollback_content = LEGACY_ROLLBACK_SCRIPT.read_text() + predeploy_backup_content = PREDEPLOY_BACKUP_SCRIPT.read_text() + + self.assertIn( + 'ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE"', + cutover_content, + ) + self.assertIn('mkdir -p "$ROLLBACK_DIR"', cutover_content) + self.assertIn('chown root:root "$ROLLBACK_DIR"', cutover_content) + self.assertIn('chmod 700 "$ROLLBACK_DIR"', cutover_content) + self.assertIn( + 'SNAPSHOT="$ROLLBACK_DIR/legacy_${OLD_SHORT}.tar.gz"', + cutover_content, + ) + self.assertIn('SNAPSHOT_TMP="$SNAPSHOT.tmp.$$"', cutover_content) + self.assertIn('tar -czf "$SNAPSHOT_TMP"', cutover_content) + self.assertIn('tar -tzf "$SNAPSHOT_TMP" >/dev/null', cutover_content) + self.assertIn('mv "$SNAPSHOT_TMP" "$SNAPSHOT"', cutover_content) + self.assertIn('chown root:root "$SNAPSHOT"', cutover_content) + self.assertNotIn( + 'chown "$INST_USER:$INST_USER" "$SNAPSHOT"', + cutover_content, + ) + self.assertLess( + cutover_content.index("for unit in gunicorn celery-worker celery-beat"), + cutover_content.index('tar -czf "$SNAPSHOT_TMP"'), + ) + + self.assertIn( + 'ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE"', + rollback_content, + ) + self.assertIn('ROLLBACK_DIR_MODE="$(stat -c', rollback_content) + self.assertIn('"0:0:700"', rollback_content) + self.assertIn( + 'SNAPSHOT_MATCHES=("$ROLLBACK_DIR"/legacy_"$OLD_PREFIX"*.tar.gz)', + rollback_content, + ) + self.assertIn( + 'UNITS_DIR="$ROLLBACK_DIR/legacy_${OLD_SHORT}.units"', + rollback_content, + ) + self.assertIn( + 'NGINX_BACKUP="$ROLLBACK_DIR/legacy_${OLD_SHORT}.nginx.conf"', + rollback_content, + ) + self.assertIn( + 'DB_MATCHES=("$ROLLBACK_DIR"/predeploy_*_"$OLD_SHORT".sql.gz)', + rollback_content, + ) + self.assertIn('DB_DUMP_MODE="$(stat -c', rollback_content) + self.assertIn( + "Legacy predeploy backup must be root:root mode 600", + rollback_content, + ) + self.assertIn('gunzip -t "$DB_DUMP"', rollback_content) + self.assertNotIn( + 'DB_MATCHES=("$BACKUP_DIR"/predeploy_*_"$OLD_SHORT".sql.gz)', + rollback_content, + ) + self.assertNotIn( + 'SNAPSHOT_MATCHES=("$BACKUP_DIR"/legacy_"$OLD_PREFIX"*.tar.gz)', + rollback_content, + ) + + self.assertIn( + 'ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE"', + predeploy_backup_content, + ) + self.assertIn( + 'LEGACY_MANIFEST="$ROLLBACK_DIR/legacy_${HASH}.manifest"', + predeploy_backup_content, + ) + self.assertIn('if [[ -f "$LEGACY_MANIFEST" ]]; then', predeploy_backup_content) + self.assertIn('OUT_DIR="$ROLLBACK_DIR"', predeploy_backup_content) + self.assertIn('chown root:root "$OUT"', predeploy_backup_content) + self.assertIn('chmod 600 "$OUT"', predeploy_backup_content) + + def test_legacy_rollback_preserves_backups_and_uses_restored_venv(self) -> None: + content = LEGACY_ROLLBACK_SCRIPT.read_text() + + self.assertNotIn('chown -R "$INST_USER:$INST_USER" "$INSTANCE_DIR"', content) + self.assertIn('[[ "$(basename "$path")" == "backups" ]] && continue', content) + self.assertIn("grep -Fx './manage.py'", content) + self.assertIn("grep -Fx './.venv/bin/python'", content) + self.assertIn('LEGACY_PYTHON="$INSTANCE_DIR/.venv/bin/python"', content) + self.assertIn('"$3" manage.py sync_sequences', content) + self.assertNotIn("SHARED_VENV", content) + + def test_deploy_uses_single_legacy_checkout_predicate(self) -> None: + content = DEPLOY_SCRIPT.read_text() + + self.assertIn("is_legacy_checkout()", content) + self.assertIn( + '[[ -d "$instance_dir/.git" && ! -L "$instance_dir/current" ]]', + content, + ) + self.assertIn('if is_legacy_checkout "$instance_dir"; then', content) + self.assertIn( + 'if [[ -n "$previous_sha" ]] && ! is_legacy_checkout "$instance_dir"; then', + content, + ) + self.assertIn( + "sudo $SCRIPT_DIR/../legacy_rollback.sh $instance ${previous_sha:0:12}", + content, + ) + self.assertIn( + "sudo $SCRIPT_DIR/../predeploy_rollback.sh $instance ${previous_sha:0:12}", + content, + ) + self.assertNotIn( + 'if [[ -n "$previous_sha" && ! -d "$instance_dir/.git" ]]', + content, + ) + + def test_server_readme_documents_both_rollback_paths(self) -> None: + content = SERVER_README.read_text() + + self.assertIn("predeploy_rollback.sh", content) + self.assertIn("legacy_rollback.sh", content) + self.assertIn("first legacy checkout cutover", content) + def test_typed_router_drift_is_checked_in_release_build(self) -> None: deploy_content = DEPLOY_SCRIPT.read_text() release_utils_content = RELEASE_UTILS.read_text() diff --git a/docs/updating.md b/docs/updating.md index 176703f9b..1eaa4c01b 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -58,8 +58,8 @@ sudo ./scripts/predeploy_rollback.sh - ``` Deploy builds the previous release before switching, so this rollback target -always exists — including the first cutover from a legacy per-instance checkout, -whose old SHA had never been built as a shared release. +always exists for an instance already on shared releases. For the first cutover +from a legacy per-instance checkout, use the standalone snapshot scripts instead. Do not switch only the `current` symlink after a migration failure; old code can be incompatible with the partially migrated database. diff --git a/scripts/cleanup_backups.py b/scripts/cleanup_backups.py old mode 100644 new mode 100755 diff --git a/scripts/legacy_rollback.sh b/scripts/legacy_rollback.sh new file mode 100755 index 000000000..0b5875300 --- /dev/null +++ b/scripts/legacy_rollback.sh @@ -0,0 +1,224 @@ +#!/bin/bash +set -euo pipefail + +# Usage: legacy_rollback.sh +# Example: legacy_rollback.sh msm-uat ea793bd1 +# +# Rolls a cutover instance back to its legacy pre-cutover state using the +# snapshot created by cutover_legacy_instance.sh and the paired predeploy +# DB backup. Restores the legacy code tree, database, systemd units, and +# nginx config — then restarts services. +# +# Must run as root. Destructive: drops the current database. + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/server/common.sh" +source "$SCRIPT_DIR/server/release-utils.sh" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: This script must be run as root (use sudo)." >&2 + exit 1 +fi + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " + echo "Example: $0 msm-uat ea793bd1" + exit 1 +fi + +INSTANCE="$1" +OLD_PREFIX="$2" +INSTANCE_DIR="$INSTANCES_DIR/$INSTANCE" +ENV_FILE="$INSTANCE_DIR/.env" +ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE" +INST_USER="$(instance_user "$INSTANCE")" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "ERROR: $ENV_FILE missing" >&2 + exit 1 +fi + +DB_NAME="$(read_env_value "$ENV_FILE" DB_NAME)" +if [[ -z "$DB_NAME" ]]; then + echo "ERROR: DB_NAME not set in $ENV_FILE" >&2 + exit 1 +fi +DB_USER="$(read_env_value "$ENV_FILE" DB_USER)" +if [[ -z "$DB_USER" ]]; then + echo "ERROR: DB_USER not set in $ENV_FILE" >&2 + exit 1 +fi + +shopt -s nullglob +if [[ ! -d "$ROLLBACK_DIR" ]]; then + echo "ERROR: Legacy rollback artifact directory missing: $ROLLBACK_DIR" >&2 + echo "Run cutover_legacy_instance.sh first to create one." >&2 + exit 1 +fi + +ROLLBACK_DIR_MODE="$(stat -c '%u:%g:%a' "$ROLLBACK_DIR")" +if [[ "$ROLLBACK_DIR_MODE" != "0:0:700" ]]; then + echo "ERROR: Legacy rollback artifact directory must be root:root mode 700: $ROLLBACK_DIR" >&2 + exit 1 +fi + +SNAPSHOT_MATCHES=("$ROLLBACK_DIR"/legacy_"$OLD_PREFIX"*.tar.gz) +if (( ${#SNAPSHOT_MATCHES[@]} == 0 )); then + echo "ERROR: Legacy snapshot not found for prefix $OLD_PREFIX in $ROLLBACK_DIR" >&2 + echo "Run cutover_legacy_instance.sh first to create one." >&2 + exit 1 +elif (( ${#SNAPSHOT_MATCHES[@]} > 1 )); then + echo "ERROR: Multiple legacy snapshots match prefix $OLD_PREFIX:" >&2 + printf ' %s\n' "${SNAPSHOT_MATCHES[@]}" >&2 + echo "Pass a longer SHA prefix." >&2 + exit 1 +fi + +SNAPSHOT="${SNAPSHOT_MATCHES[0]}" +OLD_SHORT="$(basename "$SNAPSHOT")" +OLD_SHORT="${OLD_SHORT#legacy_}" +OLD_SHORT="${OLD_SHORT%.tar.gz}" +UNITS_DIR="$ROLLBACK_DIR/legacy_${OLD_SHORT}.units" +NGINX_BACKUP="$ROLLBACK_DIR/legacy_${OLD_SHORT}.nginx.conf" + +if [[ ! -d "$UNITS_DIR" ]]; then + echo "ERROR: Legacy unit files not found: $UNITS_DIR" >&2 + exit 1 +fi +for unit in gunicorn celery-worker celery-beat; do + unit_path="$UNITS_DIR/${unit}-${INSTANCE}.service" + if [[ ! -f "$unit_path" ]]; then + echo "ERROR: Legacy unit file not found: $unit_path" >&2 + exit 1 + fi +done +if [[ ! -f "$NGINX_BACKUP" ]]; then + echo "ERROR: Legacy nginx config not found: $NGINX_BACKUP" >&2 + exit 1 +fi +if ! tar -tzf "$SNAPSHOT" >/dev/null; then + echo "ERROR: Legacy snapshot is not a readable gzip tarball: $SNAPSHOT" >&2 + exit 1 +fi +if ! tar -tzf "$SNAPSHOT" | grep -Fx './manage.py' >/dev/null; then + echo "ERROR: Legacy snapshot does not contain manage.py: $SNAPSHOT" >&2 + exit 1 +fi +if ! tar -tzf "$SNAPSHOT" | grep -Fx './.venv/bin/python' >/dev/null; then + echo "ERROR: Legacy snapshot does not contain .venv/bin/python: $SNAPSHOT" >&2 + exit 1 +fi + +# Locate the newest protected predeploy backup for this legacy SHA. +DB_MATCHES=("$ROLLBACK_DIR"/predeploy_*_"$OLD_SHORT".sql.gz) +if (( ${#DB_MATCHES[@]} == 0 )); then + echo "ERROR: No protected predeploy backup found for hash $OLD_SHORT in $ROLLBACK_DIR" >&2 + exit 1 +fi +mapfile -t SORTED_DB < <(printf '%s\n' "${DB_MATCHES[@]}" | sort) +DB_DUMP="${SORTED_DB[-1]}" +DB_DUMP_MODE="$(stat -c '%u:%g:%a' "$DB_DUMP")" +if [[ "$DB_DUMP_MODE" != "0:0:600" ]]; then + echo "ERROR: Legacy predeploy backup must be root:root mode 600: $DB_DUMP" >&2 + exit 1 +fi +if ! gunzip -t "$DB_DUMP"; then + echo "ERROR: Legacy predeploy backup is not a readable gzip file: $DB_DUMP" >&2 + exit 1 +fi + +echo "=== Rolling $INSTANCE back to legacy state at $OLD_SHORT ===" +echo "=== Code snapshot: $SNAPSHOT" +echo "=== DB restore: $DB_DUMP" +echo "=== Units dir: $UNITS_DIR" +echo "=== Nginx config: $NGINX_BACKUP" +echo "=== This will DROP the current database and restore the legacy checkout." +read -rp "Continue? [y/N] " ans +if [[ "$ans" != "y" && "$ans" != "Y" ]]; then + echo "Aborted." + exit 1 +fi + +# --- Stop services --- +log "Stopping services for $INSTANCE" +systemctl stop "celery-beat-$INSTANCE" 2>/dev/null || true +systemctl stop "celery-worker-$INSTANCE" 2>/dev/null || true +systemctl stop "gunicorn-$INSTANCE" 2>/dev/null || true + +# --- Drop and recreate the database --- +log "Dropping database $DB_NAME" +sudo -u postgres psql -v ON_ERROR_STOP=1 -v db_name="$DB_NAME" -v db_user="$DB_USER" postgres < pg_backend_pid(); +DROP DATABASE :"db_name"; +CREATE DATABASE :"db_name" OWNER :"db_user"; +EOSQL + +log "Restoring predeploy backup $DB_DUMP into $DB_NAME" +gunzip -c "$DB_DUMP" | sudo -u postgres psql -v ON_ERROR_STOP=1 "$DB_NAME" >/dev/null +log "Database restore completed" + +# --- Restore legacy code tree --- +log "Removing current release link" +rm -f "$INSTANCE_DIR/current" "$INSTANCE_DIR/deploy-state.env" + +log "Extracting legacy code tree from $SNAPSHOT" +tar -xzf "$SNAPSHOT" -C "$INSTANCE_DIR" + +chown "$INST_USER:$INST_USER" "$INSTANCE_DIR" +for path in "$INSTANCE_DIR"/* "$INSTANCE_DIR"/.[!.]* "$INSTANCE_DIR"/..?*; do + [[ -e "$path" ]] || continue + [[ "$(basename "$path")" == "backups" ]] && continue + chown -R "$INST_USER:$INST_USER" "$path" +done +log "Legacy code tree restored" + +# --- Sync database sequences (needs manage.py from restored code) --- +log "Syncing database sequences" +LEGACY_PYTHON="$INSTANCE_DIR/.venv/bin/python" +if [[ ! -x "$LEGACY_PYTHON" ]]; then + echo "ERROR: restored legacy venv python is missing or not executable: $LEGACY_PYTHON" >&2 + exit 1 +fi + +sudo -u "$INST_USER" bash -c ' + set -a + source "$1" + set +a + export DJANGO_SETTINGS_MODULE=docketworks.settings + export PYTHONDONTWRITEBYTECODE=1 + cd "$2" + "$3" manage.py sync_sequences +' _ "$ENV_FILE" "$INSTANCE_DIR" "$LEGACY_PYTHON" +log "Sequence sync complete" + +# --- Restore systemd units --- +log "Restoring legacy systemd units" +cp "$UNITS_DIR/gunicorn-$INSTANCE.service" /etc/systemd/system/ +cp "$UNITS_DIR/celery-worker-$INSTANCE.service" /etc/systemd/system/ +cp "$UNITS_DIR/celery-beat-$INSTANCE.service" /etc/systemd/system/ +systemctl daemon-reload + +# --- Restore nginx config --- +log "Restoring legacy nginx config" +cp "$NGINX_BACKUP" "/etc/nginx/sites-available/docketworks-$INSTANCE" +if ! nginx -t; then + echo "ERROR: nginx configuration test failed after restoring legacy config" >&2 + echo "The legacy code tree has been restored but nginx is NOT reloaded." >&2 + exit 1 +fi +systemctl reload nginx + +# --- Start services --- +if [[ -f "$INSTANCE_DIR/.dr-mode" ]]; then + log "DR mode (.dr-mode present): leaving services stopped" +else + log "Starting services for $INSTANCE" + systemctl start "celery-worker-$INSTANCE" + systemctl start "celery-beat-$INSTANCE" + systemctl start "gunicorn-$INSTANCE" +fi + +log "=== Legacy rollback complete: $INSTANCE running at $OLD_SHORT ===" diff --git a/scripts/predeploy_backup.sh b/scripts/predeploy_backup.sh index 6b6419a51..3d9cab894 100755 --- a/scripts/predeploy_backup.sh +++ b/scripts/predeploy_backup.sh @@ -27,6 +27,7 @@ INSTANCE="$1" INSTANCE_DIR="$INSTANCES_DIR/$INSTANCE" ENV_FILE="$INSTANCE_DIR/.env" BACKUP_DIR="$INSTANCE_DIR/backups" +ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE" if [[ ! -f "$ENV_FILE" ]]; then echo "ERROR: $ENV_FILE missing" >&2 @@ -45,9 +46,21 @@ if [[ -z "$HASH" ]]; then fi HASH="${HASH:0:12}" TS=$(date +%Y%m%d_%H%M%S) -OUT="$BACKUP_DIR/predeploy_${TS}_${HASH}.sql.gz" mkdir -p "$BACKUP_DIR" +OUT_DIR="$BACKUP_DIR" + +LEGACY_MANIFEST="$ROLLBACK_DIR/legacy_${HASH}.manifest" +if [[ -f "$LEGACY_MANIFEST" ]]; then + ROLLBACK_DIR_MODE="$(stat -c '%u:%g:%a' "$ROLLBACK_DIR")" + if [[ "$ROLLBACK_DIR_MODE" != "0:0:700" ]]; then + echo "ERROR: Legacy rollback artifact directory must be root:root mode 700: $ROLLBACK_DIR" >&2 + exit 1 + fi + OUT_DIR="$ROLLBACK_DIR" +fi + +OUT="$OUT_DIR/predeploy_${TS}_${HASH}.sql.gz" # Atomic write: dump to .tmp, rename on success. A mid-dump failure then # leaves a .tmp file (not a finished-looking predeploy_*.sql.gz). @@ -55,5 +68,9 @@ mkdir -p "$BACKUP_DIR" # would succeed on empty input. sudo -u postgres pg_dump "$DB_NAME" | gzip > "$OUT.tmp" mv "$OUT.tmp" "$OUT" +if [[ "$OUT_DIR" == "$ROLLBACK_DIR" ]]; then + chown root:root "$OUT" + chmod 600 "$OUT" +fi echo "Wrote $OUT" diff --git a/scripts/server/README.md b/scripts/server/README.md index 33f0fecc0..42ef02046 100644 --- a/scripts/server/README.md +++ b/scripts/server/README.md @@ -115,7 +115,7 @@ What `deploy.sh` does, in order: 1. Pull latest code from GitHub (into the shared local repo). 2. Run `server-setup.sh` to converge host-level deps. Cheap when nothing's missing; lands new system deps automatically when a future PR adds them. 3. Resolve the target ref to a SHA and build `/opt/docketworks/releases/` once if it does not already exist. -4. For each instance: build the previous release if it is missing (rollback target — a no-op on a normal deploy, but on a first legacy→new cutover this builds the old SHA's release so rollback has a target), take a pre-deploy backup (unless `--no-backup`), stop `celery-beat-`, `celery-worker-`, and `gunicorn-`, switch `current` to the release, run migrate, render backup units, restart `celery-worker-`, restart `celery-beat-` (the periodic-task dispatcher), restart `gunicorn-`. If migrate fails, services stay stopped and rollback is explicit via `sudo ./scripts/predeploy_rollback.sh `. Worker restarts before beat so a freshly-dispatched periodic task lands on a worker that knows the task name; gunicorn last for the same reason on webhook-dispatched tasks. +4. For each instance: build the previous release if it is missing (rollback target — a no-op on a normal deploy; skipped for legacy checkouts since their cutover rollback target is the snapshot), take a pre-deploy backup (unless `--no-backup`), stop `celery-beat-`, `celery-worker-`, and `gunicorn-`, switch `current` to the release, run migrate, render backup units, restart `celery-worker-`, restart `celery-beat-` (the periodic-task dispatcher), restart `gunicorn-`. If migrate fails, services stay stopped and rollback is explicit via `sudo ./scripts/predeploy_rollback.sh ` for shared-release instances, or `sudo ./scripts/legacy_rollback.sh ` for a first legacy checkout cutover. Worker restarts before beat so a freshly-dispatched periodic task lands on a worker that knows the task name; gunicorn last for the same reason on webhook-dispatched tasks. 5. Clean up complete releases that are no longer referenced by an instance `current` symlink or rollback state. To run only cleanup: `sudo ./scripts/server/deploy.sh --cleanup-releases`. diff --git a/scripts/server/cutover_legacy_instance.sh b/scripts/server/cutover_legacy_instance.sh new file mode 100755 index 000000000..368014615 --- /dev/null +++ b/scripts/server/cutover_legacy_instance.sh @@ -0,0 +1,152 @@ +#!/bin/bash +set -euo pipefail + +# Usage: cutover_legacy_instance.sh +# Example: cutover_legacy_instance.sh msm-uat +# +# Takes a restorable snapshot of a legacy per-instance checkout, then +# exec-s into deploy.sh for the actual cutover to shared immutable releases. +# The snapshot is the rollback target — the legacy SHA cannot be built as a +# shared release. +# +# Must run as root. One-time use per legacy instance; retire after the last +# legacy instance (msm-prod) migrates. + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/common.sh" +source "$SCRIPT_DIR/release-utils.sh" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: This script must be run as root (use sudo)." >&2 + exit 1 +fi + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + echo "Example: $0 msm-uat" + exit 1 +fi + +INSTANCE="$1" +INSTANCE_DIR="$INSTANCES_DIR/$INSTANCE" +BACKUP_DIR="$INSTANCE_DIR/backups" +ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE" +INST_USER="$(instance_user "$INSTANCE")" + +if [[ ! -d "$INSTANCE_DIR" ]]; then + echo "ERROR: Instance directory $INSTANCE_DIR does not exist." >&2 + exit 1 +fi + +if [[ ! -f "$INSTANCE_DIR/.env" ]]; then + echo "ERROR: No .env file found at $INSTANCE_DIR/.env" >&2 + exit 1 +fi + +# Assert this is a legacy instance: must have .git and NO current symlink. +if [[ ! -d "$INSTANCE_DIR/.git" ]]; then + echo "ERROR: $INSTANCE_DIR is not a legacy checkout (no .git directory)." >&2 + echo "Use 'deploy.sh $INSTANCE' for instances already on shared releases." >&2 + exit 1 +fi + +if [[ -L "$INSTANCE_DIR/current" ]]; then + echo "ERROR: $INSTANCE_DIR already has a current symlink — not a legacy instance." >&2 + echo "Use 'deploy.sh $INSTANCE' for instances already on shared releases." >&2 + exit 1 +fi + +OLD_SHA="$(sudo -u "$INST_USER" git -C "$INSTANCE_DIR" rev-parse HEAD)" +OLD_SHORT="${OLD_SHA:0:12}" +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +log "Legacy instance $INSTANCE at $OLD_SHORT — taking cutover snapshot..." + +mkdir -p "$BACKUP_DIR" +chown "$INST_USER:$INST_USER" "$BACKUP_DIR" +chmod 700 "$BACKUP_DIR" +mkdir -p "$ROLLBACK_DIR" +chown root:root "$ROLLBACK_DIR" +chmod 700 "$ROLLBACK_DIR" + +SNAPSHOT="$ROLLBACK_DIR/legacy_${OLD_SHORT}.tar.gz" +SNAPSHOT_TMP="$SNAPSHOT.tmp.$$" +UNITS_DIR="$ROLLBACK_DIR/legacy_${OLD_SHORT}.units" +NGINX_BACKUP="$ROLLBACK_DIR/legacy_${OLD_SHORT}.nginx.conf" +MANIFEST="$ROLLBACK_DIR/legacy_${OLD_SHORT}.manifest" + +cleanup_snapshot_tmp() { + rm -f "$SNAPSHOT_TMP" +} +trap cleanup_snapshot_tmp EXIT + +for unit in gunicorn celery-worker celery-beat; do + unit_path="/etc/systemd/system/${unit}-${INSTANCE}.service" + if [[ ! -f "$unit_path" ]]; then + echo "ERROR: Required unit file not found: $unit_path" >&2 + echo "Snapshot not created; run instance.sh or restore the unit before cutover." >&2 + exit 1 + fi +done + +NGINX_CONF="/etc/nginx/sites-available/docketworks-$INSTANCE" +if [[ ! -f "$NGINX_CONF" ]]; then + echo "ERROR: Required nginx config not found: $NGINX_CONF" >&2 + echo "Snapshot not created; run instance.sh or restore the config before cutover." >&2 + exit 1 +fi + +# Snapshot the full code tree including frontend/dist, frontend/dist-manual, +# and .git. Exclude data dirs that are either large regenerable runtime state +# or already backed up separately. +log " Creating code snapshot: $SNAPSHOT" +tar -czf "$SNAPSHOT_TMP" \ + -C "$INSTANCE_DIR" \ + --exclude='./dropbox' \ + --exclude='./mediafiles' \ + --exclude='./phone-recordings' \ + --exclude='./session-replays' \ + --exclude='./backups' \ + --exclude='./node_modules' \ + --exclude='./.cache' \ + --exclude='./logs' \ + . +tar -tzf "$SNAPSHOT_TMP" >/dev/null +mv "$SNAPSHOT_TMP" "$SNAPSHOT" +chown root:root "$SNAPSHOT" +chmod 600 "$SNAPSHOT" + +# Save the three systemd unit files. +log " Saving systemd unit files -> $UNITS_DIR" +rm -rf "$UNITS_DIR" +mkdir -p "$UNITS_DIR" +for unit in gunicorn celery-worker celery-beat; do + unit_path="/etc/systemd/system/${unit}-${INSTANCE}.service" + cp "$unit_path" "$UNITS_DIR/" +done +chown -R root:root "$UNITS_DIR" +chmod 700 "$UNITS_DIR" +chmod 600 "$UNITS_DIR"/* + +# Save the nginx server config. +log " Saving nginx config -> $NGINX_BACKUP" +cp "$NGINX_CONF" "$NGINX_BACKUP" +chown root:root "$NGINX_BACKUP" +chmod 600 "$NGINX_BACKUP" + +# Write manifest. +cat > "$MANIFEST" <&2 @@ -85,6 +87,15 @@ validate_instance() { echo "ERROR: $local_dir has neither current release link nor legacy git checkout." >&2 exit 1 fi + + if [[ -f "$creds_file" ]]; then + require_root_owned_credentials_file "$creds_file" + fi + + if [[ ! -f "$nginx_conf" ]]; then + echo "ERROR: nginx config not found at $nginx_conf. Run instance.sh first." >&2 + exit 1 + fi } render_runtime_units() { @@ -205,6 +216,11 @@ remove_legacy_scheduler_unit() { fi } +is_legacy_checkout() { + local instance_dir="$1" + [[ -d "$instance_dir/.git" && ! -L "$instance_dir/current" ]] +} + # --- Determine targets --- TARGETS=() if [[ $DEPLOY_ALL -eq 1 ]]; then @@ -296,7 +312,7 @@ for instance in "${TARGETS[@]}"; do log " Previous SHA: ${previous_sha:-none}" log " Target SHA: $TARGET_SHA" - if [[ -d "$instance_dir/.git" ]]; then + if is_legacy_checkout "$instance_dir"; then tree_dirty=0 if ! sudo -u "$inst_user" git -C "$instance_dir" diff --quiet --ignore-submodules HEAD --; then tree_dirty=1 @@ -317,11 +333,10 @@ for instance in "${TARGETS[@]}"; do fi # Ensure the previous release is built so predeploy_rollback.sh has a target. - # No-op on a normal deploy (its release is already complete); on a first - # legacy->new cutover this builds the old SHA's release from the shared repo, - # before anything destructive, so rollback works. Fatal if it cannot build — - # don't cut over without a rollback target. - if [[ -n "$previous_sha" ]]; then + # No-op on a normal deploy (its release is already complete). Skip legacy + # checkouts — they can't be built as shared releases (the cutover snapshot is + # the rollback target for a first migration). + if [[ -n "$previous_sha" ]] && ! is_legacy_checkout "$instance_dir"; then log " Ensuring previous release ${previous_sha:0:12} is built (rollback target)..." ensure_release "$previous_sha" fi @@ -349,7 +364,11 @@ for instance in "${TARGETS[@]}"; do if [[ -n "$previous_sha" ]]; then if [[ $DO_BACKUP -eq 1 ]]; then log " Manual rollback, if required:" - log " sudo $SCRIPT_DIR/../predeploy_rollback.sh $instance ${previous_sha:0:12}" + if is_legacy_checkout "$instance_dir"; then + log " sudo $SCRIPT_DIR/../legacy_rollback.sh $instance ${previous_sha:0:12}" + else + log " sudo $SCRIPT_DIR/../predeploy_rollback.sh $instance ${previous_sha:0:12}" + fi else log " WARNING: --no-backup was used; no pre-deploy rollback backup was created" fi