Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions apps/workflow/management/commands/rollback_migrations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
"""Move a database back to the migration leaves from an older release."""

from pathlib import Path

from django.core.management.base import BaseCommand, CommandError, CommandParser
from django.db import connection
from django.db.migrations.executor import MigrationExecutor

MigrationTarget = tuple[str, str | None]


def read_migration_targets(path: Path) -> list[tuple[str, str]]:
targets: list[tuple[str, str]] = []
for line_number, raw_line in enumerate(
path.read_text(encoding="utf-8").splitlines(), start=1
):
line = raw_line.strip()
if not line:
continue
parts = line.split("\t")
if len(parts) != 2 or not all(parts):
raise CommandError(
f"Invalid migration target at {path}:{line_number}: {raw_line!r}"
)
targets.append((parts[0], parts[1]))
if not targets:
raise CommandError(f"No migration targets found in {path}")
return targets


class Command(BaseCommand):
help = "Plan or apply reverse migrations to an older release's leaf nodes."

def add_arguments(self, parser: CommandParser) -> None:
parser.add_argument("--targets-file", required=True)
parser.add_argument("--apply", action="store_true")

def handle(self, *args: object, **options: object) -> None:
targets_file = options["targets_file"]
apply_plan = options["apply"]
if not isinstance(targets_file, str):
raise CommandError("--targets-file must be a path")
if not isinstance(apply_plan, bool):
raise CommandError("--apply must be a boolean flag")

release_targets = read_migration_targets(Path(targets_file))
target_apps = {app_label for app_label, _ in release_targets}
executor = MigrationExecutor(connection)
targets: list[MigrationTarget] = list(release_targets)
targets.extend(
(app_label, None)
for app_label in sorted(executor.loader.migrated_apps - target_apps)
)

plan = executor.migration_plan(targets)
forward_migrations = [
migration for migration, backwards in plan if not backwards
]
if forward_migrations:
names = ", ".join(
f"{migration.app_label}.{migration.name}"
for migration in forward_migrations
)
raise CommandError(
f"Target release would require forward migrations: {names}"
)

irreversible = [
migration
for migration, _ in plan
if not all(operation.reversible for operation in migration.operations)
]
if irreversible:
names = ", ".join(
f"{migration.app_label}.{migration.name}" for migration in irreversible
)
raise CommandError(f"Irreversible migrations in rollback plan: {names}")

if not plan:
self.stdout.write("No migration changes required.")
return

for migration, _ in plan:
self.stdout.write(f"UNAPPLY {migration.app_label}.{migration.name}")

if apply_plan:
executor.migrate(targets)
self.stdout.write(self.style.SUCCESS("Reverse migrations complete."))
else:
self.stdout.write("Plan only; pass --apply to execute.")
101 changes: 101 additions & 0 deletions apps/workflow/tests/test_rollback_migrations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
from io import StringIO
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import MagicMock, patch

from django.core.management.base import CommandError, OutputWrapper
from django.test import SimpleTestCase

from apps.workflow.management.commands.rollback_migrations import (
Command,
read_migration_targets,
)


class ReadMigrationTargetsTests(SimpleTestCase):
def test_reads_release_leaf_nodes(self) -> None:
with TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "targets.tsv"
path.write_text(
"company\t0007_people\njob\t0012_events\n", encoding="utf-8"
)

self.assertEqual(
read_migration_targets(path),
[("company", "0007_people"), ("job", "0012_events")],
)

def test_rejects_malformed_target(self) -> None:
with TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "targets.tsv"
path.write_text("company-only\n", encoding="utf-8")

with self.assertRaises(CommandError):
read_migration_targets(path)


class RollbackMigrationsCommandTests(SimpleTestCase):
def _targets_file(self, temp_dir: str) -> Path:
path = Path(temp_dir) / "targets.tsv"
path.write_text("company\t0007_people\n", encoding="utf-8")
return path

@patch("apps.workflow.management.commands.rollback_migrations.MigrationExecutor")
def test_empty_plan_reports_code_only_rollback(
self, executor_class: MagicMock
) -> None:
executor = executor_class.return_value
executor.loader.migrated_apps = {"company"}
executor.migration_plan.return_value = []
output = StringIO()
command = Command()
command.stdout = OutputWrapper(output)

with TemporaryDirectory() as temp_dir:
command.handle(targets_file=str(self._targets_file(temp_dir)), apply=False)

self.assertEqual(output.getvalue(), "No migration changes required.\n")
executor.migrate.assert_not_called()

@patch("apps.workflow.management.commands.rollback_migrations.MigrationExecutor")
def test_apply_executes_reversible_reverse_plan(
self, executor_class: MagicMock
) -> None:
executor = executor_class.return_value
executor.loader.migrated_apps = {"company"}
migration = MagicMock()
migration.app_label = "company"
migration.name = "0008_new_field"
migration.operations = [MagicMock(reversible=True)]
executor.migration_plan.return_value = [(migration, True)]
output = StringIO()
command = Command()
command.stdout = OutputWrapper(output)

with TemporaryDirectory() as temp_dir:
command.handle(targets_file=str(self._targets_file(temp_dir)), apply=True)

self.assertIn("UNAPPLY company.0008_new_field", output.getvalue())
executor.migrate.assert_called_once_with([("company", "0007_people")])

@patch("apps.workflow.management.commands.rollback_migrations.MigrationExecutor")
def test_irreversible_reverse_plan_is_reported(
self, executor_class: MagicMock
) -> None:
executor = executor_class.return_value
executor.loader.migrated_apps = {"company"}
migration = MagicMock()
migration.app_label = "company"
migration.name = "0008_irreversible"
migration.operations = [MagicMock(reversible=False)]
executor.migration_plan.return_value = [(migration, True)]

with (
TemporaryDirectory() as temp_dir,
self.assertRaisesMessage(
CommandError, "Irreversible migrations in rollback plan"
),
):
Command().handle(
targets_file=str(self._targets_file(temp_dir)), apply=False
)
6 changes: 6 additions & 0 deletions docs/adr/0029-servers-run-the-production-branch.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ Feature PRs target `main`. Testing and UAT servers typically track `main`;
production servers typically track `production`. After UAT verification, a
release PR promotes `main` to `production`.

Each installed instance records its tracked Git ref in its deployment state.
Deploys resolve that per-instance ref rather than relying on a global default
or an inferred environment naming convention.

A hotfix branches from `production`, merges back by PR, deploys, and is
immediately back-merged to `main`.

Expand All @@ -35,3 +39,5 @@ reviewable promotion.
- Releasing gains one explicit step: the `main` → `production` promotion PR.
- Hotfixes must be back-merged to `main` immediately.
- `production` carries the same branch protections as `main`.
- Bare deploy commands remember each instance's configured ref, including when
one `--all` run targets instances that track different branches.
52 changes: 40 additions & 12 deletions docs/updating.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,25 @@ sudo ./scripts/server/deploy.sh <client>-<env>
sudo ./scripts/server/deploy.sh --all
```

That's it for a normal code release. `deploy.sh` pulls `production` itself, builds or reuses the shared `/opt/docketworks/releases/<sha>` release, then for each target instance takes a pre-deploy DB backup, stops runtime services, switches `app` to that release, runs migrations, and restarts its services — you don't run anything per service.
That's it for a normal code release. Each instance records its tracked Git ref
alongside its current and previous commit in
`/opt/docketworks/instances/<instance>/deploy-state.env`. `deploy.sh` fetches
GitHub, resolves each target instance's ref, builds or reuses the shared
`/opt/docketworks/releases/<sha>` release, then takes a pre-deploy DB backup,
stops runtime services, switches `app` to that release, runs migrations, and
restarts its services — you don't run anything per service.

To configure an existing instance after upgrading, or to change what it tracks,
deploy once with an explicit ref:

```bash
sudo ./scripts/server/deploy.sh msm-uat --ref origin/main
sudo ./scripts/server/deploy.sh docketworks-demo --ref origin/production
```

The explicit ref is persisted with the successful deployment state. Subsequent
bare deploys remember it. A bare `--all` deploy can therefore deploy different
refs for different instances.

If migrations fail, deploy leaves that instance's services stopped and does not
perform an automatic rollback. Django records successful migrations in the
Expand All @@ -53,19 +71,28 @@ the database partially upgraded. Investigate in the failed release first:
sudo ./scripts/server/dw-run.sh <client>-<env> python manage.py showmigrations
```

If the right response is rollback rather than fix-forward, run the explicit
rollback command printed by deploy. It is printed only when deploy created a
fresh pre-deploy backup, and restores that paired database backup before
switching the instance back to the matching release:
List the releases previously installed on the instance:

```bash
sudo ./scripts/server/instance.sh history <client> <env>
```

Roll back code while retaining the latest database and applying Django reverse
migrations:

```bash
sudo ./scripts/rollback.sh <client>-<env> <previous-8-char-sha>
```

Or restore the database snapshot paired with the target release:

```bash
sudo ./scripts/predeploy_rollback.sh <client>-<env> <previous-8-char-sha>
sudo ./scripts/rollback.sh <client>-<env> <previous-8-char-sha> --restore-backup
```

Deploy builds the previous release before switching, so the code rollback target
always exists for an instance already on shared releases. If deploy was run with
`--no-backup`, no fresh paired database dump exists and deploy will warn instead
of printing a rollback command.
Both modes take a fresh safety backup first. A paired restore requires the
target release's pre-deploy backup; `--no-backup` may mean that pair does not
exist.

Do not switch only the `app` symlink after a migration failure; old code can
be incompatible with the partially migrated database.
Expand All @@ -81,8 +108,9 @@ taken at or after the squash restore and migrate normally — the ledger rides
along and the baseline migrations record themselves automatically.

Release cleanup is part of deploy. The script removes stale incomplete
`.building-*` directories at the start and removes complete releases that are no
longer referenced by any instance `app` symlink or rollback state at the end.
`.building-*` directories at the start. Complete releases are retained for 14
days after they were last activated and are never removed while referenced by
an instance `app` symlink or rollback state.
To run only the cleanup pass:

```bash
Expand Down
4 changes: 2 additions & 2 deletions scripts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ Run manually for one-off or periodic data tasks:
- **`pull_prod_files.sh`** — File-side companion to `pull_prod_backup.sh`: incrementally rsyncs a prod instance's `mediafiles`, `phone-recordings`, and `session-replays` (the same set `backup_instance_files.sh` pushes to Drive) into this checkout's local storage roots from `.env`. Files are instance-user-owned, so the remote rsync escalates via `sudo -iu <instance-user>`. A DB restore brings file paths but not the files (see [`docs/restore-prod-to-hotfix.md`](../docs/restore-prod-to-hotfix.md) step 4). Usage: `scripts/pull_prod_files.sh [host] [instance-user]` (defaults: `MSM dw_msm_prod`).
- **`pull_prod_recordings.sh`** — Incrementally rsyncs a production instance's `phone-recordings/` and `session-replays/` into this checkout's local storage roots (from `.env`). Files are instance-user-owned, so the remote rsync escalates via `sudo -iu <instance-user>`. Complements a DB restore, which brings file paths but not the files (see [`docs/restore-prod-to-hotfix.md`](../docs/restore-prod-to-hotfix.md) step 4). Usage: `scripts/pull_prod_recordings.sh [host] [instance-user]` (defaults: `MSM dw_msm_prod`).
- **`verify_scrubbed_backup.py`** — Fail-closed verifier for scrubbed custom-format PostgreSQL archives. Fully reads every archive entry, reports table row counts only, and never emits credential values.
- **`predeploy_backup.sh`** — Called by `scripts/server/deploy.sh` before each instance is switched to a new release. Stamps the dump with the current release hash so rollback is a (switch release, psql restore) pair. Runnable by hand: `sudo predeploy_backup.sh <instance>`
- **`predeploy_rollback.sh`** — Restore an instance to the release + data that paired with a given commit hash. Usage: `sudo predeploy_rollback.sh <instance> <8-char-hash>` (interactive confirm; restores the dump into a temporary DB before stopping services and swapping DBs)
- **`predeploy_backup.sh`** — Called by `scripts/server/deploy.sh` before each instance is switched to a new release. Stamps the dump with the current release hash so `rollback.sh --restore-backup` can restore the database paired with that release. Runnable by hand: `sudo predeploy_backup.sh <instance>`
- **`rollback.sh`** — Roll an instance back to a prior release. The default keeps the latest database and runs Django reverse migrations; `--restore-backup` restores the database snapshot paired with the target SHA.
Comment thread
coderabbitai[bot] marked this conversation as resolved.
- **`cleanup_backups.py`** — DB backup retention and remote upload. Copies local dumps before pruning, then purges only the matching expired remote names. Legacy `ts_dir` style: keep 24h + daily for a week + monthly beyond. `predeploy_*.sql.gz`: keep 30 days. `daily_*.sql.gz`: keep 14. `monthly_*.sql.gz`: keep 12. Other filenames left alone.
- **`cleanup_backups.sh`** — Wrapper that activates venv and runs `cleanup_backups.py`

Expand Down
Loading
Loading