From 7ad59268dec1c5cec3074720132f90b4e4ed7e05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 09:25:30 -0700 Subject: [PATCH 1/4] Retire in favor of sqlite-utils Migrations, refs #18 --- README.md | 178 +--------------- pyproject.toml | 9 +- sqlite_migrate/__init__.py | 121 +---------- sqlite_migrate/sqlite_utils_plugin.py | 115 ----------- tests/conftest.py | 3 + tests/test_sqlite_migrate.py | 8 +- tests/test_sqlite_utils_migrate_command.py | 227 ++------------------- 7 files changed, 36 insertions(+), 625 deletions(-) delete mode 100644 sqlite_migrate/sqlite_utils_plugin.py create mode 100644 tests/conftest.py diff --git a/README.md b/README.md index e1e8830..cb13b3c 100644 --- a/README.md +++ b/README.md @@ -5,188 +5,30 @@ [![Tests](https://github.com/simonw/sqlite-migrate/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-migrate/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-migrate/blob/main/LICENSE) -A simple database migration system for SQLite, based on [sqlite-utils](https://sqlite-utils.datasette.io/). +Deprecated compatibility package for `sqlite-utils` migrations. -**This project is an early alpha. Expect breaking changes.** - -## Installation - -This tool works as a plugin for `sqlite-utils`. First [install that](https://sqlite-utils.datasette.io/en/stable/installation.html): +Migration support is now built into `sqlite-utils` 4. Install and use `sqlite-utils` directly for new projects: ```bash pip install sqlite-utils ``` -Then install this plugin like so: -```bash -sqlite-utils install sqlite-migrate -``` -## Migration files -This tool works against migration files. A migration file looks like this: +New code should import `Migrations` from `sqlite_utils`: ```python -from sqlite_migrate import Migrations - -# Pick a unique name here - it must not clash with other migration sets that -# the user might run against the same database. - -migration = Migrations("creatures") - -# Use this decorator against functions that implement migrations -@migration() -def create_table(db): - # db is a sqlite-utils Database instance - db["creatures"].create( - {"id": int, "name": str, "species": str}, - pk="id" - ) - -@migration() -def add_weight(db): - # db is a sqlite-utils Database instance - db["creatures"].add_column("weight", float) +from sqlite_utils import Migrations ``` -Here is [documentation on the Database instance](https://sqlite-utils.datasette.io/en/stable/python-api.html) passed to each migration function. -## Running migrations +This package depends on `sqlite-utils>=4` and re-exports that class so existing migration files can continue to use their old import: -Running this command will execute those migrations in sequence against the specified database file. - -Call `migrate` with a path to your database and a path to the migrations file you want to apply: -```bash -sqlite-utils migrate creatures.db path/to/migrations.py -``` -Running this multiple times will have no additional affect, unless you add more migration functions to the file. - -If you call it without arguments it will search for and apply any `migrations.py` files in the current directory or any of its subdirectories. - -You can also pass the path to a directory, in which case all `migrations.py` files in that directory and its subdirectories will be applied: - -```bash -sqlite-utils migrate creatures.db path/to/parent/ -``` -When applying a single migrations file you can use the `--stop-before` option to apply all migrations up to but excluding the specified migration: - -```bash -sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight -``` - -## Listing migrations - -Add `--list` to list migrations without running them, for example: - -```bash -sqlite-utils migrate creatures.db --list -``` -The output will look something like this: -``` -Migrations for: creatures - - Applied: - create_table - 2023-07-23 04:09:40.324002 - add_weight - 2023-07-23 04:09:40.324649 - add_age - 2023-07-23 04:09:44.441616 - cleanup_columns - 2023-07-23 04:09:44.443394 - - Pending: - drop_table +```python +from sqlite_migrate import Migrations ``` -## Verbose mode - -Add `-v` or `--verbose` for verbose output, which will show the schema before and after the migrations were applied along with a diff: +Run migrations using the `sqlite-utils migrate` command: ```bash -sqlite-utils migrate creatures.db --verbose -``` -Example output: - - +sqlite-utils migrate creatures.db path/to/migrations.py ``` -Migrating creatures.db - -Schema before: - CREATE TABLE [_sqlite_migrations] ( - [id] INTEGER PRIMARY KEY, - [migration_set] TEXT, - [name] TEXT, - [applied_at] TEXT - ); - CREATE UNIQUE INDEX [idx__sqlite_migrations_migration_set_name] - ON [_sqlite_migrations] ([migration_set], [name]); - CREATE TABLE [creatures] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [species] TEXT, - [weight] FLOAT - ); - -Schema after: - - CREATE TABLE [_sqlite_migrations] ( - [id] INTEGER PRIMARY KEY, - [migration_set] TEXT, - [name] TEXT, - [applied_at] TEXT - ); - CREATE UNIQUE INDEX [idx__sqlite_migrations_migration_set_name] - ON [_sqlite_migrations] ([migration_set], [name]); - CREATE TABLE "creatures" ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [species] TEXT, - [weight] FLOAT, - [age] INTEGER, - [shoe_size] INTEGER - ); - -Schema diff: - - ); - CREATE UNIQUE INDEX [idx__sqlite_migrations_migration_set_name] - ON [_sqlite_migrations] ([migration_set], [name]); --CREATE TABLE [creatures] ( -+CREATE TABLE "creatures" ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [species] TEXT, -- [weight] FLOAT -+ [weight] FLOAT, -+ [age] INTEGER, -+ [shoe_size] INTEGER - ); -``` - +See the `sqlite-utils` migrations documentation for the full Python API and CLI usage. diff --git a/pyproject.toml b/pyproject.toml index 79a7ecc..cb3c3bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,17 @@ [project] name = "sqlite-migrate" version = "0.1b0" -description = "A simple database migration system for SQLite, based on sqlite-utils" +description = "Compatibility package for sqlite-utils migrations" readme = "README.md" requires-python = ">=3.10" authors = [{name = "Simon Willison"}] license = "Apache-2.0" classifiers = [ - "Development Status :: 2 - Pre-Alpha" + "Development Status :: 7 - Inactive" ] dependencies = [ - "sqlite-utils!=4.0,<4.0.post0" + "sqlite-utils>=4" ] [project.urls] @@ -20,9 +20,6 @@ Changelog = "https://github.com/simonw/sqlite-migrate/releases" Issues = "https://github.com/simonw/sqlite-migrate/issues" CI = "https://github.com/simonw/sqlite-migrate/actions" -[project.entry-points.sqlite_utils] -migrate = "sqlite_migrate.sqlite_utils_plugin" - [dependency-groups] dev = ["pytest", "mypy", "black", "ruff", "cogapp"] diff --git a/sqlite_migrate/__init__.py b/sqlite_migrate/__init__.py index 9cb5400..111ce8d 100644 --- a/sqlite_migrate/__init__.py +++ b/sqlite_migrate/__init__.py @@ -1,120 +1,3 @@ -from dataclasses import dataclass -import datetime -from typing import cast, Callable, List, Optional -from typing import TYPE_CHECKING +from sqlite_utils import Migrations -if TYPE_CHECKING: - from sqlite_utils.db import Database, Table - - -class Migrations: - migrations_table = "_sqlite_migrations" - - @dataclass - class _Migration: - name: str - fn: Callable - - @dataclass - class _AppliedMigration: - name: str - applied_at: datetime.datetime - - def __init__(self, name: str): - """ - :param name: The name of the migration set. This should be unique. - """ - self.name = name - self._migrations: List[Migrations._Migration] = [] - - def __call__(self, *, name: Optional[str] = None) -> Callable: - """ - :param name: The name to use for this migration - if not provided, - the name of the function will be used - """ - - def inner(func: Callable) -> Callable: - self._migrations.append(self._Migration(name or func.__name__, func)) - return func - - return inner - - def pending(self, db: "Database") -> List["Migrations._Migration"]: - """ - Return a list of pending migrations. - """ - self.ensure_migrations_table(db) - already_applied = { - r["name"] - for r in db[self.migrations_table].rows_where( - "migration_set = ?", [self.name] - ) - } - return [ - migration - for migration in self._migrations - if migration.name not in already_applied - ] - - def applied(self, db: "Database") -> List["Migrations._AppliedMigration"]: - """ - Return a list of applied migrations. - """ - self.ensure_migrations_table(db) - return [ - self._AppliedMigration(name=row["name"], applied_at=row["applied_at"]) - for row in db[self.migrations_table].rows_where( - "migration_set = ?", [self.name] - ) - ] - - def apply(self, db: "Database", *, stop_before: Optional[str] = None): - """ - Apply any pending migrations to the database. - """ - self.ensure_migrations_table(db) - for migration in self.pending(db): - name = migration.name - if name == stop_before: - return - migration.fn(db) - _table(db, self.migrations_table).insert( - { - "migration_set": self.name, - "name": name, - "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), - } - ) - - def ensure_migrations_table(self, db: "Database"): - """ - Ensure _sqlite_migrations table exists and has the correct schema - """ - table = _table(db, self.migrations_table) - if not table.exists(): - table.create( - { - "id": int, - "migration_set": str, - "name": str, - "applied_at": str, - }, - pk="id", - ) - table.create_index(["migration_set", "name"], unique=True) - elif table.pks != ["id"]: - # This has an older primary key scheme, upgrade it - table.transform(pk="id") - unique_indexes = {tuple(index.columns) for index in table.indexes} - if ("migration_set", "name") not in unique_indexes: - table.create_index(["migration_set", "name"], unique=True) - - def __repr__(self): - return "".format( - self.name, ", ".join(m.name for m in self._migrations) - ) - - -def _table(db: "Database", name: str) -> "Table": - # mypy workaround - return cast("Table", db[name]) +__all__ = ["Migrations"] diff --git a/sqlite_migrate/sqlite_utils_plugin.py b/sqlite_migrate/sqlite_utils_plugin.py deleted file mode 100644 index 4ed4699..0000000 --- a/sqlite_migrate/sqlite_utils_plugin.py +++ /dev/null @@ -1,115 +0,0 @@ -import click -import difflib -import pathlib -import sqlite_utils -from sqlite_migrate import Migrations -import textwrap - - -@sqlite_utils.hookimpl -def register_commands(cli): - @cli.command() - @click.argument( - "db_path", type=click.Path(dir_okay=False, readable=True, writable=True) - ) - @click.argument("migrations", type=click.Path(dir_okay=True, exists=True), nargs=-1) - @click.option("--stop-before", help="Stop before applying this migration") - @click.option( - "list_", "--list", is_flag=True, help="List migrations without running them" - ) - @click.option("-v", "--verbose", is_flag=True, help="Show verbose output") - def migrate(db_path, migrations, stop_before, list_, verbose): - """ - Apply pending database migrations. - - Usage: - - sqlite-utils migrate database.db - - This will find the migrations.py file in the current directory - or subdirectories and apply any pending migrations. - - Or pass paths to one or more migrations.py files directly: - - sqlite-utils migrate database.db path/to/migrations.py - - Pass --list to see a list of applied and pending migrations - without applying them. - """ - if not migrations: - # Scan current directory for migrations.py files - migrations = [pathlib.Path(".").resolve()] - files = set() - for path_str in migrations: - path = pathlib.Path(path_str) - if path.is_dir(): - files.update(path.rglob("migrations.py")) - else: - files.add(path) - migration_sets = [] - for filepath in files: - code = filepath.read_text() - namespace = {} - exec(code, namespace) - # Find all instances of Migrations - for obj in namespace.values(): - if isinstance(obj, Migrations): - migration_sets.append(obj) - if not migration_sets: - raise click.ClickException("No migrations.py files found") - - if stop_before and len(migration_sets) > 1: - raise click.ClickException( - "--stop-before can only be used with a single migrations.py file" - ) - - db = sqlite_utils.Database(db_path) - - if list_: - display_list(db, migration_sets) - return - - prev_schema = db.schema - if verbose: - click.echo("Migrating {}".format(db_path)) - click.echo("\nSchema before:\n") - click.echo(textwrap.indent(prev_schema, " ") or " (empty)") - click.echo() - for migration_set in migration_sets: - migration_set.apply(db, stop_before=stop_before) - if verbose: - click.echo("Schema after:\n") - post_schema = db.schema - if post_schema == prev_schema: - click.echo(" (unchanged)") - else: - click.echo(textwrap.indent(post_schema, " ")) - click.echo("\nSchema diff:\n") - # Calculate and display a diff - diff = list( - difflib.unified_diff( - prev_schema.splitlines(), post_schema.splitlines() - ) - ) - # Skipping the first two lines since they only make - # sense if we provided filenames, and the next one - # because it is just @@ -0,0 +1,15 @@ - click.echo("\n".join(diff[3:])) - - -def display_list(db, migration_sets): - for migration_set in migration_sets: - print("Migrations for: {}".format(migration_set.name)) - print() - print(" Applied:") - for migration in migration_set.applied(db): - print(" {} - {}".format(migration.name, migration.applied_at)) - print() - print(" Pending:") - output = False - for migration in migration_set.pending(db): - output = True - print(" {}".format(migration.name)) - if not output: - print(" (none)") - print() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1d2c4f6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,3 @@ +import sys + +setattr(sys, "_called_from_test", True) diff --git a/tests/test_sqlite_migrate.py b/tests/test_sqlite_migrate.py index 53f58d4..94096a7 100644 --- a/tests/test_sqlite_migrate.py +++ b/tests/test_sqlite_migrate.py @@ -3,6 +3,10 @@ import pytest +def test_reexports_sqlite_utils_migrations(): + assert Migrations is sqlite_utils.Migrations + + @pytest.fixture def migrations(): migrations = Migrations("test") @@ -14,7 +18,7 @@ def m001(db): @migrations() def m002(db): db["cats"].create({"name": str}) - db.query("insert into dogs (name) values ('Pancakes')") + db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -32,7 +36,7 @@ def m002(db): @migrations() def m001(db): db["cats"].create({"name": str}) - db.query("insert into dogs (name) values ('Pancakes')") + db.execute("insert into dogs (name) values ('Pancakes')") return migrations diff --git a/tests/test_sqlite_utils_migrate_command.py b/tests/test_sqlite_utils_migrate_command.py index 2c1af5c..2c681f1 100644 --- a/tests/test_sqlite_utils_migrate_command.py +++ b/tests/test_sqlite_utils_migrate_command.py @@ -1,234 +1,31 @@ -import sqlite_utils.cli import pathlib -from click.testing import CliRunner -import pytest -from sqlite_migrate import Migrations -from sqlite_migrate.sqlite_utils_plugin import display_list - -TWO_MIGRATIONS = """ -from sqlite_migrate import Migrations - -m = Migrations("hello") - -@m() -def foo(db): - db["foo"].insert({"hello": "world"}) - -@m() -def bar(db): - db["bar"].insert({"hello": "world"}) -""" - - -@pytest.fixture -def two_migrations(tmpdir): - path = pathlib.Path(tmpdir) - (path / "foo").mkdir() - migrations_py = path / "foo" / "migrations.py" - migrations_py.write_text(TWO_MIGRATIONS, "utf-8") - return path, migrations_py - - -@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/")) -def test_basic(two_migrations, arg): - path, _ = two_migrations - db_path = str(path / "test.db") - - runner = CliRunner() - - def _list(): - list_result = runner.invoke( - sqlite_utils.cli.cli, - ["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))], - ) - assert list_result.exit_code == 0 - return list_result.output - - assert _list() == ( - "Migrations for: hello\n\n" - " Applied:\n\n" - " Pending:\n" - " foo\n" - " bar\n\n" - ) - - result = runner.invoke( - sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))] - ) - assert result.exit_code == 0, result.output - - list_output = _list() - assert "Migrations for: hello\n\n Applied:\n " in list_output - prior_to_pending = list_output.split(" Pending")[0] - assert " foo" in prior_to_pending - assert " bar" in prior_to_pending - assert " Pending:\n (none)" in list_output - - db = sqlite_utils.Database(db_path) - assert db["foo"].exists() - assert db["bar"].exists() - assert db["_sqlite_migrations"].exists() - rows = list(db["_sqlite_migrations"].rows) - assert len(rows) == 2 - assert rows[0]["name"] == "foo" - assert rows[1]["name"] == "bar" - - -def test_list_same_migration_names_in_different_sets(capsys): - applied = Migrations("applied") - - @applied() - def foo(db): - db["applied"].insert({"hello": "world"}) - - pending = Migrations("pending") - @pending() - def foo(db): - db["pending"].insert({"hello": "world"}) - - db = sqlite_utils.Database(memory=True) - applied.apply(db) - - display_list(db, [applied, pending]) - - output = capsys.readouterr().out - assert ( - "Migrations for: pending\n\n" " Applied:\n\n" " Pending:\n" " foo\n\n" - ) in output +from click.testing import CliRunner +import sqlite_utils +import sqlite_utils.cli -def test_verbose(tmpdir): +def test_sqlite_utils_migrate_command_with_sqlite_migrate_import(tmpdir): path = pathlib.Path(tmpdir) - (path / "foo").mkdir() - migrations_py = path / "foo" / "migrations.py" + migrations_py = path / "migrations.py" migrations_py.write_text( """ from sqlite_migrate import Migrations -m = Migrations("hello") +migrations = Migrations("hello") -@m() +@migrations() def foo(db): - db["dogs"].insert({"id": 1, "name": "Cleo"}) - """, + db["foo"].insert({"hello": "world"}) +""", "utf-8", ) db_path = str(path / "test.db") - runner = CliRunner() - result = runner.invoke( - sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)] - ) - assert result.exit_code == 0 - # Now run again with --verbose, should be no changes - result = runner.invoke( - sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] - ) - assert result.exit_code == 0 - expected = """ -Schema before: - - CREATE TABLE [_sqlite_migrations] ( - [id] INTEGER PRIMARY KEY, - [migration_set] TEXT, - [name] TEXT, - [applied_at] TEXT - ); - CREATE UNIQUE INDEX [idx__sqlite_migrations_migration_set_name] - ON [_sqlite_migrations] ([migration_set], [name]); - CREATE TABLE [dogs] ( - [id] INTEGER, - [name] TEXT - ); - -Schema after: - - (unchanged) -""".strip() - assert expected in result.output - # Now append to the migration and run it - new_migration = """ -@m() -def bar(db): - db["dogs"].add_column("age", int) - db["dogs"].add_column("weight", float) - db["dogs"].transform() -""" - # Append that to migrations.py - migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration) - # And run it - result = runner.invoke( - sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + result = CliRunner().invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)] ) - assert result.exit_code == 0 - expected_diff = """ -Schema diff: - - ); - CREATE UNIQUE INDEX [idx__sqlite_migrations_migration_set_name] - ON [_sqlite_migrations] ([migration_set], [name]); --CREATE TABLE [dogs] ( -+CREATE TABLE "dogs" ( - [id] INTEGER, -- [name] TEXT -+ [name] TEXT, -+ [age] INTEGER, -+ [weight] FLOAT - ); -""".strip() - assert expected_diff in result.output - -def test_stop_before(two_migrations): - path, _ = two_migrations - db_path = str(path / "test.db") - runner = CliRunner() - result = runner.invoke( - sqlite_utils.cli.cli, - [ - "migrate", - db_path, - str(path / "foo" / "migrations.py"), - "--stop-before", - "bar", - ], - ) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) assert db["foo"].exists() - assert not db["bar"].exists() - - -def test_stop_before_error(two_migrations): - path, _ = two_migrations - db_path = str(path / "test.db") - (path / "foo" / "migrations2.py").write_text( - """ -from sqlite_migrate import Migrations - -m = Migrations("hello2") - -@m() -def foo(db): - db["foo"].insert({"hello": "world"}) - """, - "utf-8", - ) - runner = CliRunner() - result = runner.invoke( - sqlite_utils.cli.cli, - [ - "migrate", - db_path, - str(path / "foo" / "migrations.py"), - str(path / "foo" / "migrations2.py"), - "--stop-before", - "foo", - ], - ) - assert result.exit_code == 1 - assert ( - "--stop-before can only be used with a single migrations.py file" - in result.output - ) From 570e23a7086eb3753b5502c58b5c57ca9ef2f5f8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 09:26:34 -0700 Subject: [PATCH 2/4] No need for conftest.py --- tests/conftest.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 1d2c4f6..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,3 +0,0 @@ -import sys - -setattr(sys, "_called_from_test", True) From d7d273fffc98c4c844f423d6fca7b8a51979a72f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 09:29:16 -0700 Subject: [PATCH 3/4] Link to sqlite-utils migrations docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb13b3c..59caa2d 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,4 @@ Run migrations using the `sqlite-utils migrate` command: sqlite-utils migrate creatures.db path/to/migrations.py ``` -See the `sqlite-utils` migrations documentation for the full Python API and CLI usage. +See the [`sqlite-utils` migrations documentation](https://sqlite-utils.datasette.io/en/stable/migrations.html) for the full Python API and CLI usage. From 4219a50f700a1170d3ee1d44dcbd8781d01b2023 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 09:30:07 -0700 Subject: [PATCH 4/4] pip install . --group dev --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 49d96f0..781b6c9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,7 +23,7 @@ jobs: cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install '.[test]' + pip install . --group dev - name: Run tests run: | pytest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 33b91ad..01b44a8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install '.[test]' + pip install . --group dev - name: Run tests run: | pytest