feat: use Alembic to initialize the database

This completely removes the previous DB patching mechanism.
When this is first run for an existing installation of Unwind, depending
on its version it might lead to problems because the database's schema
won't match the code.
To avoid that issue, when upgrading Unwind to this version make sure to
STOP the old application, install this new version but DON'T start it,
instead use `alembic upgrade head` to run the outstanding patches, and
only then start the application.

alembic/env.py

@@ -91,13 +91,18 @@ async def run_async_migrations() -> None:
     await connectable.dispose()
 
 
-def run_migrations_online_async() -> None:
+def run_migrations_online() -> None:
     """Run migrations in 'online' mode."""
-
+    # Support having a (sync) connection passed in from another script.
-    asyncio.run(run_async_migrations())
+    if (conn := config.attributes.get("connection")) and isinstance(
+        conn, sa.Connection
+    ):
+        do_run_migrations(conn)
+    else:
+        asyncio.run(run_async_migrations())
 
 
 if context.is_offline_mode():
     run_migrations_offline()
 else:
-    run_migrations_online_async()
+    run_migrations_online()

alembic/versions/1716077466-8b06e4916840_remove_db_patches_table.py

@@ -0,0 +1,38 @@
+"""remove db_patches table
+
+We replace our old patch process with Alembic's.
+
+Revision ID: 8b06e4916840
+Revises: f17c7ca9afa4
+Create Date: 2024-05-19 00:11:06.730421+00:00
+
+"""
+
+from typing import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "8b06e4916840"
+down_revision: str | None = "f17c7ca9afa4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("db_patches")
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "db_patches",
+        sa.Column("id", sa.INTEGER(), nullable=False),
+        sa.Column("current", sa.VARCHAR(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###

tests/test_db.py

@@ -20,14 +20,6 @@ def a_movie(**kwds) -> models.Movie:
     return models.Movie(**args)
 
 
-@pytest.mark.asyncio
-async def test_current_patch_level(conn: db.Connection):
-    patch_level = "some-patch-level"
-    assert patch_level != await db.current_patch_level(conn)
-    await db.set_current_patch_level(conn, patch_level)
-    assert patch_level == await db.current_patch_level(conn)
-
-
 @pytest.mark.asyncio
 async def test_get(conn: db.Connection):
     m1 = a_movie()

unwind/db.py

@@ -4,9 +4,12 @@ from pathlib import Path
 from typing import Any, AsyncGenerator, Iterable, Literal, Sequence, Type, TypeVar
 
 import sqlalchemy as sa
-from sqlalchemy.dialects.sqlite import insert
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
 
+import alembic.command
+import alembic.config
+import alembic.migration
+
 from . import config
 from .models import (
     Model,
@@ -15,7 +18,6 @@ from .models import (
     Rating,
     User,
     asplain,
-    db_patches,
     fromplain,
     metadata,
     movies,
@@ -33,6 +35,25 @@ _engine: AsyncEngine | None = None
 
 type Connection = AsyncConnection
 
+_project_dir = Path(__file__).parent.parent
+_alembic_ini = _project_dir / "alembic.ini"
+
+
+def _init(conn: sa.Connection) -> None:
+    # See https://alembic.sqlalchemy.org/en/latest/cookbook.html#building-an-up-to-date-database-from-scratch
+    context = alembic.migration.MigrationContext.configure(conn)
+    heads = context.get_current_heads()
+
+    is_empty_db = not heads  # We consider a DB empty if Alembic hasn't touched it yet.
+    if is_empty_db:
+        log.info("⚡️ Initializing empty database.")
+        metadata.create_all(conn)
+
+        # We pass our existing connection to Alembic's env.py, to avoid running another asyncio loop there.
+        alembic_cfg = alembic.config.Config(_alembic_ini)
+        alembic_cfg.attributes["connection"] = conn
+        alembic.command.stamp(alembic_cfg, "head")
+
 
 async def open_connection_pool() -> None:
     """Open the DB connection pool.
@@ -41,11 +62,7 @@ async def open_connection_pool() -> None:
     """
     async with transaction() as conn:
         await conn.execute(sa.text("PRAGMA journal_mode=WAL"))
-
+        await conn.run_sync(_init)
-        await conn.run_sync(metadata.create_all, tables=[db_patches])
-
-    async with new_connection() as conn:
-        await apply_db_patches(conn)
 
 
 async def close_connection_pool() -> None:
@@ -65,65 +82,7 @@ async def close_connection_pool() -> None:
     await engine.dispose()
 
 
-async def current_patch_level(conn: Connection, /) -> str:
+async def vacuum(conn: Connection, /) -> None:
-    query = sa.select(db_patches.c.current)
-    current = await conn.scalar(query)
-    return current or ""
-
-
-async def set_current_patch_level(conn: Connection, /, current: str) -> None:
-    stmt = insert(db_patches).values(id=1, current=current)
-    stmt = stmt.on_conflict_do_update(set_={"current": stmt.excluded.current})
-    await conn.execute(stmt)
-
-
-db_patches_dir = Path(__file__).parent / "sql"
-
-
-async def apply_db_patches(conn: Connection, /) -> None:
-    """Apply all remaining patches to the database.
-
-    Beware that patches will be applied in lexicographical order,
-    i.e. "10" comes before "9".
-
-    The current patch state is recorded in the DB itself.
-
-    Please note that every SQL statement in a patch file MUST be terminated
-    using two consecutive semi-colons (;).
-    Failing to do so will result in an error.
-    """
-    applied_lvl = await current_patch_level(conn)
-
-    did_patch = False
-
-    for patchfile in sorted(db_patches_dir.glob("*.sql"), key=lambda p: p.stem):
-        patch_lvl = patchfile.stem
-        if patch_lvl <= applied_lvl:
-            continue
-
-        log.info("Applying patch: %s", patch_lvl)
-
-        sql = patchfile.read_text()
-        queries = sql.split(";;")
-        if len(queries) < 2:
-            log.error(
-                "Patch file is missing statement terminator (`;;'): %s", patchfile
-            )
-            raise RuntimeError("No statement found.")
-
-        async with transacted(conn):
-            for query in queries:
-                await conn.execute(sa.text(query))
-
-            await set_current_patch_level(conn, patch_lvl)
-
-            did_patch = True
-
-    if did_patch:
-        await _vacuum(conn)
-
-
-async def _vacuum(conn: Connection, /) -> None:
     """Vacuum the database.
 
     This function cannot be run on a connection with an open transaction.

unwind/models.py

@@ -255,23 +255,6 @@ def utcnow() -> datetime:
     return datetime.now(timezone.utc)
 
 
-@mapper_registry.mapped
-@dataclass
-class DbPatch:
-    __table__: ClassVar[Table] = Table(
-        "db_patches",
-        metadata,
-        Column("id", Integer, primary_key=True),
-        Column("current", String),
-    )
-
-    id: int
-    current: str
-
-
-db_patches = DbPatch.__table__
-
-
 @mapper_registry.mapped
 @dataclass
 class Progress:

unwind/sql/00000000-init-0.sql

@@ -1,36 +0,0 @@
-PRAGMA foreign_keys = ON;;
-
-CREATE TABLE IF NOT EXISTS users (
-    id TEXT NOT NULL PRIMARY KEY,
-    imdb_id TEXT NOT NULL UNIQUE,
-    name TEXT NOT NULL
-);;
-
-CREATE TABLE IF NOT EXISTS movies (
-    id TEXT NOT NULL PRIMARY KEY,
-    title TEXT NOT NULL,
-    release_year NUMBER NOT NULL,
-    media_type TEXT NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    score NUMBER NOT NULL,
-    runtime NUMBER,
-    genres TEXT NOT NULL,
-    updated TEXT NOT NULL
-);;
-
-CREATE TABLE IF NOT EXISTS ratings (
-    id TEXT NOT NULL PRIMARY KEY,
-    movie_id TEXT NOT NULL,
-    user_id TEXT NOT NULL,
-    score NUMBER NOT NULL,
-    rating_date TEXT NOT NULL,
-    favorite NUMBER,
-    finished NUMBER,
-    FOREIGN KEY(movie_id) REFERENCES movies(id),
-    FOREIGN KEY(user_id) REFERENCES users(id)
-);;
-
-CREATE UNIQUE INDEX IF NOT EXISTS ratings_index ON ratings (
-    movie_id,
-    user_id
-);;

unwind/sql/00000000-init-1.sql

@@ -1,40 +0,0 @@
--- add original_title to movies table
-
--- see https://www.sqlite.org/lang_altertable.html#caution
--- 1. Create new table
--- 2. Copy data
--- 3. Drop old table
--- 4. Rename new into old 
-
-CREATE TABLE _migrate_movies (
-    id TEXT NOT NULL PRIMARY KEY,
-    title TEXT NOT NULL,
-    original_title TEXT,
-    release_year NUMBER NOT NULL,
-    media_type TEXT NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    score NUMBER,
-    runtime NUMBER,
-    genres TEXT NOT NULL,
-    updated TEXT NOT NULL
-);;
-
-INSERT INTO _migrate_movies
-SELECT
-    id,
-    title,
-    NULL,
-    release_year,
-    media_type,
-    imdb_id,
-    score,
-    runtime,
-    genres,
-    updated
-FROM movies
-WHERE true;;
-
-DROP TABLE movies;;
-
-ALTER TABLE _migrate_movies
-RENAME TO movies;;

unwind/sql/00000000-init-2.sql

@@ -1,46 +0,0 @@
--- only set original_title if it differs from title,
--- and normalize media_type with an extra table.
-
-CREATE TABLE mediatypes (
-    id INTEGER PRIMARY KEY NOT NULL,
-    name TEXT NOT NULL UNIQUE
-);;
-
-INSERT INTO mediatypes (name)
-SELECT DISTINCT media_type
-FROM movies
-WHERE true;;
-
-CREATE TABLE _migrate_movies (
-    id TEXT PRIMARY KEY NOT NULL,
-    title TEXT NOT NULL,
-    original_title TEXT,
-    release_year INTEGER NOT NULL,
-    media_type_id INTEGER NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    score INTEGER,
-    runtime INTEGER,
-    genres TEXT NOT NULL,
-    updated TEXT NOT NULL,
-    FOREIGN KEY(media_type_id) REFERENCES mediatypes(id)
-);;
-
-INSERT INTO _migrate_movies
-SELECT
-    id,
-    title,
-    (CASE WHEN original_title=title THEN NULL ELSE original_title END),
-    release_year,
-    (SELECT id FROM mediatypes WHERE name=media_type) AS media_type_id,
-    imdb_id,
-    score,
-    runtime,
-    genres,
-    updated
-FROM movies
-WHERE true;;
-
-DROP TABLE movies;;
-
-ALTER TABLE _migrate_movies
-RENAME TO movies;;

unwind/sql/00000000-init-3.sql

@@ -1,62 +0,0 @@
--- add convenient view for movies
-
-CREATE VIEW IF NOT EXISTS movies_view
-AS SELECT
-    movies.id,
-    movies.title,
-    movies.original_title,
-    movies.release_year,
-    mediatypes.name AS media_type,
-    movies.imdb_id,
-    movies.score,
-    movies.runtime,
-    movies.genres,
-    movies.updated
-FROM movies
-JOIN mediatypes ON mediatypes.id=movies.media_type_id;;
-
-CREATE TRIGGER IF NOT EXISTS insert_movies_view
-    INSTEAD OF INSERT
-    ON movies_view
-BEGIN
-    INSERT INTO movies (
-        id,
-        title,
-        original_title,
-        release_year,
-        media_type_id,
-        imdb_id,
-        score,
-        runtime,
-        genres,
-        updated
-    ) VALUES (
-        NEW.id,
-        NEW.title,
-        NEW.original_title,
-        NEW.release_year,
-        (SELECT id FROM mediatypes WHERE name=NEW.media_type),
-        NEW.imdb_id,
-        NEW.score,
-        NEW.runtime,
-        NEW.genres,
-        NEW.updated
-    );
-END;;
-
-CREATE TRIGGER IF NOT EXISTS update_movies_view
-    INSTEAD OF UPDATE OF media_type
-    ON movies_view
-BEGIN
-    UPDATE movies
-    SET media_type_id=(SELECT id FROM mediatypes WHERE name=NEW.media_type)
-    WHERE id=OLD.id;
-END;;
-
-CREATE TRIGGER IF NOT EXISTS delete_movies_view
-    INSTEAD OF DELETE
-    ON movies_view
-BEGIN
-    DELETE FROM movies
-    WHERE movies.id=OLD.id;
-END;;

unwind/sql/00000000-init-4.sql

@@ -1,37 +0,0 @@
--- denormalize movie media_type
-
-CREATE TABLE _migrate_movies (
-    id TEXT PRIMARY KEY NOT NULL,
-    title TEXT NOT NULL,
-    original_title TEXT,
-    release_year INTEGER NOT NULL,
-    media_type TEXT NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    score INTEGER,
-    runtime INTEGER,
-    genres TEXT NOT NULL,
-    updated TEXT NOT NULL
-);;
-
-INSERT INTO _migrate_movies
-SELECT
-    id,
-    title,
-    original_title,
-    release_year,
-    (SELECT name FROM mediatypes WHERE id=media_type_id) AS media_type,
-    imdb_id,
-    score,
-    runtime,
-    genres,
-    updated
-FROM movies
-WHERE true;;
-
-DROP VIEW movies_view;;
-DROP TABLE mediatypes;;
-
-DROP TABLE movies;;
-
-ALTER TABLE _migrate_movies
-RENAME TO movies;;

unwind/sql/00000001-fix-db.sql.disabled

@@ -1,2 +0,0 @@
--- see the commit of this file for details.
-;;

unwind/sql/20210705-224139.sql

@@ -1,8 +0,0 @@
--- add groups table
-
-CREATE TABLE groups (
-    id TEXT PRIMARY KEY NOT NULL,
-    name TEXT NOT NULL,
-    secret TEXT NOT NULL,
-    users TEXT NOT NULL -- JSON array
-);;

unwind/sql/20210711-172808--progress-table.sql

@@ -1,7 +0,0 @@
--- add progress table
-
-CREATE TABLE progress (
-    id TEXT PRIMARY KEY NOT NULL,
-    state TEXT NOT NULL,
-    started TEXT NOT NULL
-);;

unwind/sql/20210720-213416.sql

@@ -1,36 +0,0 @@
--- add IMDb vote count
-
-CREATE TABLE _migrate_movies (
-    id TEXT PRIMARY KEY NOT NULL,
-    title TEXT NOT NULL,
-    original_title TEXT,
-    release_year INTEGER NOT NULL,
-    media_type TEXT NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    imdb_score INTEGER,
-    imdb_votes INTEGER,
-    runtime INTEGER,
-    genres TEXT NOT NULL,
-    updated TEXT NOT NULL
-);;
-
-INSERT INTO _migrate_movies
-SELECT
-    id,
-    title,
-    original_title,
-    release_year,
-    media_type,
-    imdb_id,
-    score AS imdb_score,
-    NULL AS imdb_votes,
-    runtime,
-    genres,
-    updated
-FROM movies
-WHERE true;;
-
-DROP TABLE movies;;
-
-ALTER TABLE _migrate_movies
-RENAME TO movies;;

unwind/sql/20210720-223416.sql

@@ -1,24 +0,0 @@
--- add IMDb vote count
-
-CREATE TABLE _migrate_progress (
-    id TEXT PRIMARY KEY NOT NULL,
-    type TEXT NOT NULL,
-    state TEXT NOT NULL,
-    started TEXT NOT NULL,
-    stopped TEXT
-);;
-
-INSERT INTO _migrate_progress
-SELECT
-    id,
-    'import-imdb-movies' AS type,
-    state,
-    started,
-    NULL AS stopped
-FROM progress
-WHERE true;;
-
-DROP TABLE progress;;
-
-ALTER TABLE _migrate_progress
-RENAME TO progress;;

unwind/sql/20210721-213417.sql

@@ -1,38 +0,0 @@
--- add creation timestamp to movies
-
-CREATE TABLE _migrate_movies (
-    id TEXT PRIMARY KEY NOT NULL,
-    title TEXT NOT NULL,
-    original_title TEXT,
-    release_year INTEGER NOT NULL,
-    media_type TEXT NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    imdb_score INTEGER,
-    imdb_votes INTEGER,
-    runtime INTEGER,
-    genres TEXT NOT NULL,
-    created TEXT NOT NULL,
-    updated TEXT NOT NULL
-);;
-
-INSERT INTO _migrate_movies
-SELECT
-    id,
-    title,
-    original_title,
-    release_year,
-    media_type,
-    imdb_id,
-    imdb_score,
-    imdb_votes,
-    runtime,
-    genres,
-    updated AS created,
-    updated
-FROM movies
-WHERE true;;
-
-DROP TABLE movies;;
-
-ALTER TABLE _migrate_movies
-RENAME TO movies;;

unwind/sql/20210728-223416.sql

@@ -1,24 +0,0 @@
--- add IMDb vote count
-
-CREATE TABLE _migrate_progress (
-    id TEXT PRIMARY KEY NOT NULL,
-    type TEXT NOT NULL,
-    state TEXT NOT NULL,
-    started TEXT NOT NULL,
-    stopped TEXT
-);;
-
-INSERT INTO _migrate_progress
-SELECT
-    id,
-    type,
-    '{"percent":' || state || '}' AS state,
-    started,
-    stopped
-FROM progress
-WHERE true;;
-
-DROP TABLE progress;;
-
-ALTER TABLE _migrate_progress
-RENAME TO progress;;

unwind/sql/20210801-201151--add-user-secret.sql

@@ -1,22 +0,0 @@
--- add secret to users
-
-CREATE TABLE _migrate_users (
-    id TEXT PRIMARY KEY NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    name TEXT NOT NULL,
-    secret TEXT NOT NULL
-);;
-
-INSERT INTO _migrate_users
-SELECT
-    id,
-    imdb_id,
-    name,
-    '' AS secret
-FROM users
-WHERE true;;
-
-DROP TABLE users;;
-
-ALTER TABLE _migrate_users
-RENAME TO users;;

unwind/sql/20210802-212312--add-group-admins.sql

@@ -1,45 +0,0 @@
--- add group admins
-
---- remove secrets from groups
-CREATE TABLE _migrate_groups (
-    id TEXT PRIMARY KEY NOT NULL,
-    name TEXT NOT NULL,
-    users TEXT NOT NULL -- JSON array
-);;
-
-INSERT INTO _migrate_groups
-SELECT
-    id,
-    name,
-    users
-FROM groups
-WHERE true;;
-
-DROP TABLE groups;;
-
-ALTER TABLE _migrate_groups
-RENAME TO groups;;
-
---- add group access to users
-CREATE TABLE _migrate_users (
-    id TEXT PRIMARY KEY NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    name TEXT NOT NULL,
-    secret TEXT NOT NULL,
-    groups TEXT NOT NULL -- JSON array
-);;
-
-INSERT INTO _migrate_users
-SELECT
-    id,
-    imdb_id,
-    name,
-    secret,
-    '[]' AS groups
-FROM users
-WHERE true;;
-
-DROP TABLE users;;
-
-ALTER TABLE _migrate_users
-RENAME TO users;;

unwind/sql/20240511-001949--remove-genres-notnull.sql

@@ -1,38 +0,0 @@
--- remove NOTNULL constraint from movies.genres
-
-CREATE TABLE _migrate_movies (
-    id TEXT PRIMARY KEY NOT NULL,
-    title TEXT NOT NULL,
-    original_title TEXT,
-    release_year INTEGER NOT NULL,
-    media_type TEXT NOT NULL,
-    imdb_id TEXT NOT NULL UNIQUE,
-    imdb_score INTEGER,
-    imdb_votes INTEGER,
-    runtime INTEGER,
-    genres TEXT,
-    created TEXT NOT NULL,
-    updated TEXT NOT NULL
-);;
-
-INSERT INTO _migrate_movies
-SELECT
-    id,
-    title,
-    original_title,
-    release_year,
-    media_type,
-    imdb_id,
-    imdb_score,
-    imdb_votes,
-    runtime,
-    genres,
-    created,
-    updated
-FROM movies
-WHERE true;;
-
-DROP TABLE movies;;
-
-ALTER TABLE _migrate_movies
-RENAME TO movies;;