fix(deploy): harden legacy cutover rollback

apps/workflow/tests/test_xero_instance_templates.py

@@ -14,6 +14,9 @@
 )
 INSTANCE_SCRIPT = REPO_ROOT / "scripts" / "server" / "instance.sh"
 DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "server" / "deploy.sh"
+CUTOVER_LEGACY_SCRIPT = REPO_ROOT / "scripts" / "server" / "cutover_legacy_instance.sh"
+LEGACY_ROLLBACK_SCRIPT = REPO_ROOT / "scripts" / "legacy_rollback.sh"
+PREDEPLOY_BACKUP_SCRIPT = REPO_ROOT / "scripts" / "predeploy_backup.sh"
 COMMON_SCRIPT = REPO_ROOT / "scripts" / "server" / "common.sh"
 SERVER_SETUP_SCRIPT = REPO_ROOT / "scripts" / "server" / "server-setup.sh"
 SERVER_README = REPO_ROOT / "scripts" / "server" / "README.md"
@@ -225,13 +228,13 @@ def test_release_utils_builds_immutable_release_artifacts(self) -> None:
 
         self.assertIn("RELEASES_DIR", content)
         self.assertIn("git -C '$LOCAL_REPO' archive '$sha'", content)
-        self.assertIn("printf '%s\\n' '$sha' > '$tmp_dir/.release-sha'", content)
-        self.assertIn("python3.12 -m venv '$tmp_dir/.venv'", content)
+        self.assertIn("printf '%s\\n' '$sha' > '$release_dir/.release-sha'", content)
+        self.assertIn("python3.12 -m venv '$release_dir/.venv'", content)
         self.assertIn("npm run check:typed-router", content)
         self.assertIn("npm run build", content)
         self.assertIn("npm run manual:build", content)
         self.assertIn("rm -rf node_modules", content)
-        self.assertIn("touch '$tmp_dir/.complete'", content)
+        self.assertIn("touch '$release_dir/.complete'", content)
 
     def test_runtime_templates_use_current_release(self) -> None:
         for template in [
@@ -390,7 +393,7 @@ def test_release_cleanup_is_deploy_integrated_and_reachability_based(self) -> No
         release_utils_content = RELEASE_UTILS.read_text()
 
         self.assertIn("--cleanup-releases", deploy_content)
-        self.assertIn("cleanup_stale_release_builds", deploy_content)
+        self.assertIn("cleanup_incomplete_releases", deploy_content)
         self.assertIn('cleanup_unreferenced_releases "$TARGET_SHA"', deploy_content)
         self.assertIn("release_is_referenced()", release_utils_content)
         self.assertIn(
@@ -399,6 +402,130 @@ def test_release_cleanup_is_deploy_integrated_and_reachability_based(self) -> No
         )
         self.assertIn("Removing unreferenced release", release_utils_content)
 
+    def test_legacy_cutover_rollback_artifacts_are_root_trusted(self) -> None:
+        cutover_content = CUTOVER_LEGACY_SCRIPT.read_text()
+        rollback_content = LEGACY_ROLLBACK_SCRIPT.read_text()
+        predeploy_backup_content = PREDEPLOY_BACKUP_SCRIPT.read_text()
+
+        self.assertIn(
+            'ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE"',
+            cutover_content,
+        )
+        self.assertIn('mkdir -p "$ROLLBACK_DIR"', cutover_content)
+        self.assertIn('chown root:root "$ROLLBACK_DIR"', cutover_content)
+        self.assertIn('chmod 700 "$ROLLBACK_DIR"', cutover_content)
+        self.assertIn(
+            'SNAPSHOT="$ROLLBACK_DIR/legacy_${OLD_SHORT}.tar.gz"',
+            cutover_content,
+        )
+        self.assertIn('SNAPSHOT_TMP="$SNAPSHOT.tmp.$$"', cutover_content)
+        self.assertIn('tar -czf "$SNAPSHOT_TMP"', cutover_content)
+        self.assertIn('tar -tzf "$SNAPSHOT_TMP" >/dev/null', cutover_content)
+        self.assertIn('mv "$SNAPSHOT_TMP" "$SNAPSHOT"', cutover_content)
+        self.assertIn('chown root:root "$SNAPSHOT"', cutover_content)
+        self.assertNotIn(
+            'chown "$INST_USER:$INST_USER" "$SNAPSHOT"',
+            cutover_content,
+        )
+        self.assertLess(
+            cutover_content.index("for unit in gunicorn celery-worker celery-beat"),
+            cutover_content.index('tar -czf "$SNAPSHOT_TMP"'),
+        )
+
+        self.assertIn(
+            'ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE"',
+            rollback_content,
+        )
+        self.assertIn('ROLLBACK_DIR_MODE="$(stat -c', rollback_content)
+        self.assertIn('"0:0:700"', rollback_content)
+        self.assertIn(
+            'SNAPSHOT_MATCHES=("$ROLLBACK_DIR"/legacy_"$OLD_PREFIX"*.tar.gz)',
+            rollback_content,
+        )
+        self.assertIn(
+            'UNITS_DIR="$ROLLBACK_DIR/legacy_${OLD_SHORT}.units"',
+            rollback_content,
+        )
+        self.assertIn(
+            'NGINX_BACKUP="$ROLLBACK_DIR/legacy_${OLD_SHORT}.nginx.conf"',
+            rollback_content,
+        )
+        self.assertIn(
+            'DB_MATCHES=("$ROLLBACK_DIR"/predeploy_*_"$OLD_SHORT".sql.gz)',
+            rollback_content,
+        )
+        self.assertIn('DB_DUMP_MODE="$(stat -c', rollback_content)
+        self.assertIn(
+            "Legacy predeploy backup must be root:root mode 600",
+            rollback_content,
+        )
+        self.assertIn('gunzip -t "$DB_DUMP"', rollback_content)
+        self.assertNotIn(
+            'DB_MATCHES=("$BACKUP_DIR"/predeploy_*_"$OLD_SHORT".sql.gz)',
+            rollback_content,
+        )
+        self.assertNotIn(
+            'SNAPSHOT_MATCHES=("$BACKUP_DIR"/legacy_"$OLD_PREFIX"*.tar.gz)',
+            rollback_content,
+        )
+
+        self.assertIn(
+            'ROLLBACK_DIR="$CONFIG_DIR/legacy-rollbacks/$INSTANCE"',
+            predeploy_backup_content,
+        )
+        self.assertIn(
+            'LEGACY_MANIFEST="$ROLLBACK_DIR/legacy_${HASH}.manifest"',
+            predeploy_backup_content,
+        )
+        self.assertIn('if [[ -f "$LEGACY_MANIFEST" ]]; then', predeploy_backup_content)
+        self.assertIn('OUT_DIR="$ROLLBACK_DIR"', predeploy_backup_content)
+        self.assertIn('chown root:root "$OUT"', predeploy_backup_content)
+        self.assertIn('chmod 600 "$OUT"', predeploy_backup_content)
+
+    def test_legacy_rollback_preserves_backups_and_uses_restored_venv(self) -> None:
+        content = LEGACY_ROLLBACK_SCRIPT.read_text()
+
+        self.assertNotIn('chown -R "$INST_USER:$INST_USER" "$INSTANCE_DIR"', content)
+        self.assertIn('[[ "$(basename "$path")" == "backups" ]] && continue', content)
+        self.assertIn("grep -Fx './manage.py'", content)
+        self.assertIn("grep -Fx './.venv/bin/python'", content)
+        self.assertIn('LEGACY_PYTHON="$INSTANCE_DIR/.venv/bin/python"', content)
+        self.assertIn('"$3" manage.py sync_sequences', content)
+        self.assertNotIn("SHARED_VENV", content)
+
+    def test_deploy_uses_single_legacy_checkout_predicate(self) -> None:
+        content = DEPLOY_SCRIPT.read_text()
+
+        self.assertIn("is_legacy_checkout()", content)
+        self.assertIn(
+            '[[ -d "$instance_dir/.git" && ! -L "$instance_dir/current" ]]',
+            content,
+        )
+        self.assertIn('if is_legacy_checkout "$instance_dir"; then', content)
+        self.assertIn(
+            'if [[ -n "$previous_sha" ]] && ! is_legacy_checkout "$instance_dir"; then',
+            content,
+        )
+        self.assertIn(
+            "sudo $SCRIPT_DIR/../legacy_rollback.sh $instance ${previous_sha:0:12}",
+            content,
+        )
+        self.assertIn(
+            "sudo $SCRIPT_DIR/../predeploy_rollback.sh $instance ${previous_sha:0:12}",
+            content,
+        )
+        self.assertNotIn(
+            'if [[ -n "$previous_sha" && ! -d "$instance_dir/.git" ]]',
+            content,
+        )
+
+    def test_server_readme_documents_both_rollback_paths(self) -> None:
+        content = SERVER_README.read_text()
+
+        self.assertIn("predeploy_rollback.sh", content)
+        self.assertIn("legacy_rollback.sh", content)
+        self.assertIn("first legacy checkout cutover", content)
+
     def test_typed_router_drift_is_checked_in_release_build(self) -> None:
         deploy_content = DEPLOY_SCRIPT.read_text()
         release_utils_content = RELEASE_UTILS.read_text()

docs/updating.md

@@ -58,8 +58,8 @@ sudo ./scripts/predeploy_rollback.sh <client>-<env> <previous-sha>
 ```
 
 Deploy builds the previous release before switching, so this rollback target
-always exists — including the first cutover from a legacy per-instance checkout,
-whose old SHA had never been built as a shared release.
+always exists for an instance already on shared releases. For the first cutover
+from a legacy per-instance checkout, use the standalone snapshot scripts instead.
 
 Do not switch only the `current` symlink after a migration failure; old code can
 be incompatible with the partially migrated database.