diff --git a/alembic/env.py b/alembic/env.py index 23056ae..b3ea427 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -91,13 +91,18 @@ async def run_async_migrations() -> None: await connectable.dispose() -def run_migrations_online_async() -> None: +def run_migrations_online() -> None: """Run migrations in 'online' mode.""" - - asyncio.run(run_async_migrations()) + # Support having a (sync) connection passed in from another script. + if (conn := config.attributes.get("connection")) and isinstance( + conn, sa.Connection + ): + do_run_migrations(conn) + else: + asyncio.run(run_async_migrations()) if context.is_offline_mode(): run_migrations_offline() else: - run_migrations_online_async() + run_migrations_online() diff --git a/alembic/versions/1716077466-8b06e4916840_remove_db_patches_table.py b/alembic/versions/1716077466-8b06e4916840_remove_db_patches_table.py new file mode 100644 index 0000000..840cb33 --- /dev/null +++ b/alembic/versions/1716077466-8b06e4916840_remove_db_patches_table.py @@ -0,0 +1,38 @@ +"""remove db_patches table + +We replace our old patch process with Alembic's. + +Revision ID: 8b06e4916840 +Revises: f17c7ca9afa4 +Create Date: 2024-05-19 00:11:06.730421+00:00 + +""" + +from typing import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "8b06e4916840" +down_revision: str | None = "f17c7ca9afa4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("db_patches") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "db_patches", + sa.Column("id", sa.INTEGER(), nullable=False), + sa.Column("current", sa.VARCHAR(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### diff --git a/tests/test_db.py b/tests/test_db.py index 981e65b..c22359d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -20,14 +20,6 @@ def a_movie(**kwds) -> models.Movie: return models.Movie(**args) -@pytest.mark.asyncio -async def test_current_patch_level(conn: db.Connection): - patch_level = "some-patch-level" - assert patch_level != await db.current_patch_level(conn) - await db.set_current_patch_level(conn, patch_level) - assert patch_level == await db.current_patch_level(conn) - - @pytest.mark.asyncio async def test_get(conn: db.Connection): m1 = a_movie() diff --git a/unwind/db.py b/unwind/db.py index b609c59..38b0107 100644 --- a/unwind/db.py +++ b/unwind/db.py @@ -4,9 +4,12 @@ from pathlib import Path from typing import Any, AsyncGenerator, Iterable, Literal, Sequence, Type, TypeVar import sqlalchemy as sa -from sqlalchemy.dialects.sqlite import insert from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine +import alembic.command +import alembic.config +import alembic.migration + from . import config from .models import ( Model, @@ -15,7 +18,6 @@ from .models import ( Rating, User, asplain, - db_patches, fromplain, metadata, movies, @@ -33,6 +35,25 @@ _engine: AsyncEngine | None = None type Connection = AsyncConnection +_project_dir = Path(__file__).parent.parent +_alembic_ini = _project_dir / "alembic.ini" + + +def _init(conn: sa.Connection) -> None: + # See https://alembic.sqlalchemy.org/en/latest/cookbook.html#building-an-up-to-date-database-from-scratch + context = alembic.migration.MigrationContext.configure(conn) + heads = context.get_current_heads() + + is_empty_db = not heads # We consider a DB empty if Alembic hasn't touched it yet. + if is_empty_db: + log.info("⚡️ Initializing empty database.") + metadata.create_all(conn) + + # We pass our existing connection to Alembic's env.py, to avoid running another asyncio loop there. + alembic_cfg = alembic.config.Config(_alembic_ini) + alembic_cfg.attributes["connection"] = conn + alembic.command.stamp(alembic_cfg, "head") + async def open_connection_pool() -> None: """Open the DB connection pool. @@ -41,11 +62,7 @@ async def open_connection_pool() -> None: """ async with transaction() as conn: await conn.execute(sa.text("PRAGMA journal_mode=WAL")) - - await conn.run_sync(metadata.create_all, tables=[db_patches]) - - async with new_connection() as conn: - await apply_db_patches(conn) + await conn.run_sync(_init) async def close_connection_pool() -> None: @@ -65,65 +82,7 @@ async def close_connection_pool() -> None: await engine.dispose() -async def current_patch_level(conn: Connection, /) -> str: - query = sa.select(db_patches.c.current) - current = await conn.scalar(query) - return current or "" - - -async def set_current_patch_level(conn: Connection, /, current: str) -> None: - stmt = insert(db_patches).values(id=1, current=current) - stmt = stmt.on_conflict_do_update(set_={"current": stmt.excluded.current}) - await conn.execute(stmt) - - -db_patches_dir = Path(__file__).parent / "sql" - - -async def apply_db_patches(conn: Connection, /) -> None: - """Apply all remaining patches to the database. - - Beware that patches will be applied in lexicographical order, - i.e. "10" comes before "9". - - The current patch state is recorded in the DB itself. - - Please note that every SQL statement in a patch file MUST be terminated - using two consecutive semi-colons (;). - Failing to do so will result in an error. - """ - applied_lvl = await current_patch_level(conn) - - did_patch = False - - for patchfile in sorted(db_patches_dir.glob("*.sql"), key=lambda p: p.stem): - patch_lvl = patchfile.stem - if patch_lvl <= applied_lvl: - continue - - log.info("Applying patch: %s", patch_lvl) - - sql = patchfile.read_text() - queries = sql.split(";;") - if len(queries) < 2: - log.error( - "Patch file is missing statement terminator (`;;'): %s", patchfile - ) - raise RuntimeError("No statement found.") - - async with transacted(conn): - for query in queries: - await conn.execute(sa.text(query)) - - await set_current_patch_level(conn, patch_lvl) - - did_patch = True - - if did_patch: - await _vacuum(conn) - - -async def _vacuum(conn: Connection, /) -> None: +async def vacuum(conn: Connection, /) -> None: """Vacuum the database. This function cannot be run on a connection with an open transaction. diff --git a/unwind/models.py b/unwind/models.py index a2ffffb..07b81fb 100644 --- a/unwind/models.py +++ b/unwind/models.py @@ -255,23 +255,6 @@ def utcnow() -> datetime: return datetime.now(timezone.utc) -@mapper_registry.mapped -@dataclass -class DbPatch: - __table__: ClassVar[Table] = Table( - "db_patches", - metadata, - Column("id", Integer, primary_key=True), - Column("current", String), - ) - - id: int - current: str - - -db_patches = DbPatch.__table__ - - @mapper_registry.mapped @dataclass class Progress: diff --git a/unwind/sql/00000000-init-0.sql b/unwind/sql/00000000-init-0.sql deleted file mode 100644 index d0bd446..0000000 --- a/unwind/sql/00000000-init-0.sql +++ /dev/null @@ -1,36 +0,0 @@ -PRAGMA foreign_keys = ON;; - -CREATE TABLE IF NOT EXISTS users ( - id TEXT NOT NULL PRIMARY KEY, - imdb_id TEXT NOT NULL UNIQUE, - name TEXT NOT NULL -);; - -CREATE TABLE IF NOT EXISTS movies ( - id TEXT NOT NULL PRIMARY KEY, - title TEXT NOT NULL, - release_year NUMBER NOT NULL, - media_type TEXT NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - score NUMBER NOT NULL, - runtime NUMBER, - genres TEXT NOT NULL, - updated TEXT NOT NULL -);; - -CREATE TABLE IF NOT EXISTS ratings ( - id TEXT NOT NULL PRIMARY KEY, - movie_id TEXT NOT NULL, - user_id TEXT NOT NULL, - score NUMBER NOT NULL, - rating_date TEXT NOT NULL, - favorite NUMBER, - finished NUMBER, - FOREIGN KEY(movie_id) REFERENCES movies(id), - FOREIGN KEY(user_id) REFERENCES users(id) -);; - -CREATE UNIQUE INDEX IF NOT EXISTS ratings_index ON ratings ( - movie_id, - user_id -);; diff --git a/unwind/sql/00000000-init-1.sql b/unwind/sql/00000000-init-1.sql deleted file mode 100644 index 85d40a6..0000000 --- a/unwind/sql/00000000-init-1.sql +++ /dev/null @@ -1,40 +0,0 @@ --- add original_title to movies table - --- see https://www.sqlite.org/lang_altertable.html#caution --- 1. Create new table --- 2. Copy data --- 3. Drop old table --- 4. Rename new into old - -CREATE TABLE _migrate_movies ( - id TEXT NOT NULL PRIMARY KEY, - title TEXT NOT NULL, - original_title TEXT, - release_year NUMBER NOT NULL, - media_type TEXT NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - score NUMBER, - runtime NUMBER, - genres TEXT NOT NULL, - updated TEXT NOT NULL -);; - -INSERT INTO _migrate_movies -SELECT - id, - title, - NULL, - release_year, - media_type, - imdb_id, - score, - runtime, - genres, - updated -FROM movies -WHERE true;; - -DROP TABLE movies;; - -ALTER TABLE _migrate_movies -RENAME TO movies;; diff --git a/unwind/sql/00000000-init-2.sql b/unwind/sql/00000000-init-2.sql deleted file mode 100644 index 68fad70..0000000 --- a/unwind/sql/00000000-init-2.sql +++ /dev/null @@ -1,46 +0,0 @@ --- only set original_title if it differs from title, --- and normalize media_type with an extra table. - -CREATE TABLE mediatypes ( - id INTEGER PRIMARY KEY NOT NULL, - name TEXT NOT NULL UNIQUE -);; - -INSERT INTO mediatypes (name) -SELECT DISTINCT media_type -FROM movies -WHERE true;; - -CREATE TABLE _migrate_movies ( - id TEXT PRIMARY KEY NOT NULL, - title TEXT NOT NULL, - original_title TEXT, - release_year INTEGER NOT NULL, - media_type_id INTEGER NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - score INTEGER, - runtime INTEGER, - genres TEXT NOT NULL, - updated TEXT NOT NULL, - FOREIGN KEY(media_type_id) REFERENCES mediatypes(id) -);; - -INSERT INTO _migrate_movies -SELECT - id, - title, - (CASE WHEN original_title=title THEN NULL ELSE original_title END), - release_year, - (SELECT id FROM mediatypes WHERE name=media_type) AS media_type_id, - imdb_id, - score, - runtime, - genres, - updated -FROM movies -WHERE true;; - -DROP TABLE movies;; - -ALTER TABLE _migrate_movies -RENAME TO movies;; diff --git a/unwind/sql/00000000-init-3.sql b/unwind/sql/00000000-init-3.sql deleted file mode 100644 index 98380c7..0000000 --- a/unwind/sql/00000000-init-3.sql +++ /dev/null @@ -1,62 +0,0 @@ --- add convenient view for movies - -CREATE VIEW IF NOT EXISTS movies_view -AS SELECT - movies.id, - movies.title, - movies.original_title, - movies.release_year, - mediatypes.name AS media_type, - movies.imdb_id, - movies.score, - movies.runtime, - movies.genres, - movies.updated -FROM movies -JOIN mediatypes ON mediatypes.id=movies.media_type_id;; - -CREATE TRIGGER IF NOT EXISTS insert_movies_view - INSTEAD OF INSERT - ON movies_view -BEGIN - INSERT INTO movies ( - id, - title, - original_title, - release_year, - media_type_id, - imdb_id, - score, - runtime, - genres, - updated - ) VALUES ( - NEW.id, - NEW.title, - NEW.original_title, - NEW.release_year, - (SELECT id FROM mediatypes WHERE name=NEW.media_type), - NEW.imdb_id, - NEW.score, - NEW.runtime, - NEW.genres, - NEW.updated - ); -END;; - -CREATE TRIGGER IF NOT EXISTS update_movies_view - INSTEAD OF UPDATE OF media_type - ON movies_view -BEGIN - UPDATE movies - SET media_type_id=(SELECT id FROM mediatypes WHERE name=NEW.media_type) - WHERE id=OLD.id; -END;; - -CREATE TRIGGER IF NOT EXISTS delete_movies_view - INSTEAD OF DELETE - ON movies_view -BEGIN - DELETE FROM movies - WHERE movies.id=OLD.id; -END;; diff --git a/unwind/sql/00000000-init-4.sql b/unwind/sql/00000000-init-4.sql deleted file mode 100644 index 984ef37..0000000 --- a/unwind/sql/00000000-init-4.sql +++ /dev/null @@ -1,37 +0,0 @@ --- denormalize movie media_type - -CREATE TABLE _migrate_movies ( - id TEXT PRIMARY KEY NOT NULL, - title TEXT NOT NULL, - original_title TEXT, - release_year INTEGER NOT NULL, - media_type TEXT NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - score INTEGER, - runtime INTEGER, - genres TEXT NOT NULL, - updated TEXT NOT NULL -);; - -INSERT INTO _migrate_movies -SELECT - id, - title, - original_title, - release_year, - (SELECT name FROM mediatypes WHERE id=media_type_id) AS media_type, - imdb_id, - score, - runtime, - genres, - updated -FROM movies -WHERE true;; - -DROP VIEW movies_view;; -DROP TABLE mediatypes;; - -DROP TABLE movies;; - -ALTER TABLE _migrate_movies -RENAME TO movies;; diff --git a/unwind/sql/00000001-fix-db.sql.disabled b/unwind/sql/00000001-fix-db.sql.disabled deleted file mode 100644 index e6376a8..0000000 --- a/unwind/sql/00000001-fix-db.sql.disabled +++ /dev/null @@ -1,2 +0,0 @@ --- see the commit of this file for details. -;; diff --git a/unwind/sql/20210705-224139.sql b/unwind/sql/20210705-224139.sql deleted file mode 100644 index e714b4e..0000000 --- a/unwind/sql/20210705-224139.sql +++ /dev/null @@ -1,8 +0,0 @@ --- add groups table - -CREATE TABLE groups ( - id TEXT PRIMARY KEY NOT NULL, - name TEXT NOT NULL, - secret TEXT NOT NULL, - users TEXT NOT NULL -- JSON array -);; diff --git a/unwind/sql/20210711-172808--progress-table.sql b/unwind/sql/20210711-172808--progress-table.sql deleted file mode 100644 index 1ee6a5f..0000000 --- a/unwind/sql/20210711-172808--progress-table.sql +++ /dev/null @@ -1,7 +0,0 @@ --- add progress table - -CREATE TABLE progress ( - id TEXT PRIMARY KEY NOT NULL, - state TEXT NOT NULL, - started TEXT NOT NULL -);; diff --git a/unwind/sql/20210720-213416.sql b/unwind/sql/20210720-213416.sql deleted file mode 100644 index 286e094..0000000 --- a/unwind/sql/20210720-213416.sql +++ /dev/null @@ -1,36 +0,0 @@ --- add IMDb vote count - -CREATE TABLE _migrate_movies ( - id TEXT PRIMARY KEY NOT NULL, - title TEXT NOT NULL, - original_title TEXT, - release_year INTEGER NOT NULL, - media_type TEXT NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - imdb_score INTEGER, - imdb_votes INTEGER, - runtime INTEGER, - genres TEXT NOT NULL, - updated TEXT NOT NULL -);; - -INSERT INTO _migrate_movies -SELECT - id, - title, - original_title, - release_year, - media_type, - imdb_id, - score AS imdb_score, - NULL AS imdb_votes, - runtime, - genres, - updated -FROM movies -WHERE true;; - -DROP TABLE movies;; - -ALTER TABLE _migrate_movies -RENAME TO movies;; diff --git a/unwind/sql/20210720-223416.sql b/unwind/sql/20210720-223416.sql deleted file mode 100644 index 95e1b78..0000000 --- a/unwind/sql/20210720-223416.sql +++ /dev/null @@ -1,24 +0,0 @@ --- add IMDb vote count - -CREATE TABLE _migrate_progress ( - id TEXT PRIMARY KEY NOT NULL, - type TEXT NOT NULL, - state TEXT NOT NULL, - started TEXT NOT NULL, - stopped TEXT -);; - -INSERT INTO _migrate_progress -SELECT - id, - 'import-imdb-movies' AS type, - state, - started, - NULL AS stopped -FROM progress -WHERE true;; - -DROP TABLE progress;; - -ALTER TABLE _migrate_progress -RENAME TO progress;; diff --git a/unwind/sql/20210721-213417.sql b/unwind/sql/20210721-213417.sql deleted file mode 100644 index 33e891a..0000000 --- a/unwind/sql/20210721-213417.sql +++ /dev/null @@ -1,38 +0,0 @@ --- add creation timestamp to movies - -CREATE TABLE _migrate_movies ( - id TEXT PRIMARY KEY NOT NULL, - title TEXT NOT NULL, - original_title TEXT, - release_year INTEGER NOT NULL, - media_type TEXT NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - imdb_score INTEGER, - imdb_votes INTEGER, - runtime INTEGER, - genres TEXT NOT NULL, - created TEXT NOT NULL, - updated TEXT NOT NULL -);; - -INSERT INTO _migrate_movies -SELECT - id, - title, - original_title, - release_year, - media_type, - imdb_id, - imdb_score, - imdb_votes, - runtime, - genres, - updated AS created, - updated -FROM movies -WHERE true;; - -DROP TABLE movies;; - -ALTER TABLE _migrate_movies -RENAME TO movies;; diff --git a/unwind/sql/20210728-223416.sql b/unwind/sql/20210728-223416.sql deleted file mode 100644 index 1581060..0000000 --- a/unwind/sql/20210728-223416.sql +++ /dev/null @@ -1,24 +0,0 @@ --- add IMDb vote count - -CREATE TABLE _migrate_progress ( - id TEXT PRIMARY KEY NOT NULL, - type TEXT NOT NULL, - state TEXT NOT NULL, - started TEXT NOT NULL, - stopped TEXT -);; - -INSERT INTO _migrate_progress -SELECT - id, - type, - '{"percent":' || state || '}' AS state, - started, - stopped -FROM progress -WHERE true;; - -DROP TABLE progress;; - -ALTER TABLE _migrate_progress -RENAME TO progress;; diff --git a/unwind/sql/20210801-201151--add-user-secret.sql b/unwind/sql/20210801-201151--add-user-secret.sql deleted file mode 100644 index 3294a56..0000000 --- a/unwind/sql/20210801-201151--add-user-secret.sql +++ /dev/null @@ -1,22 +0,0 @@ --- add secret to users - -CREATE TABLE _migrate_users ( - id TEXT PRIMARY KEY NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - secret TEXT NOT NULL -);; - -INSERT INTO _migrate_users -SELECT - id, - imdb_id, - name, - '' AS secret -FROM users -WHERE true;; - -DROP TABLE users;; - -ALTER TABLE _migrate_users -RENAME TO users;; diff --git a/unwind/sql/20210802-212312--add-group-admins.sql b/unwind/sql/20210802-212312--add-group-admins.sql deleted file mode 100644 index 13f3105..0000000 --- a/unwind/sql/20210802-212312--add-group-admins.sql +++ /dev/null @@ -1,45 +0,0 @@ --- add group admins - ---- remove secrets from groups -CREATE TABLE _migrate_groups ( - id TEXT PRIMARY KEY NOT NULL, - name TEXT NOT NULL, - users TEXT NOT NULL -- JSON array -);; - -INSERT INTO _migrate_groups -SELECT - id, - name, - users -FROM groups -WHERE true;; - -DROP TABLE groups;; - -ALTER TABLE _migrate_groups -RENAME TO groups;; - ---- add group access to users -CREATE TABLE _migrate_users ( - id TEXT PRIMARY KEY NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - secret TEXT NOT NULL, - groups TEXT NOT NULL -- JSON array -);; - -INSERT INTO _migrate_users -SELECT - id, - imdb_id, - name, - secret, - '[]' AS groups -FROM users -WHERE true;; - -DROP TABLE users;; - -ALTER TABLE _migrate_users -RENAME TO users;; diff --git a/unwind/sql/20240511-001949--remove-genres-notnull.sql b/unwind/sql/20240511-001949--remove-genres-notnull.sql deleted file mode 100644 index 98a7c16..0000000 --- a/unwind/sql/20240511-001949--remove-genres-notnull.sql +++ /dev/null @@ -1,38 +0,0 @@ --- remove NOTNULL constraint from movies.genres - -CREATE TABLE _migrate_movies ( - id TEXT PRIMARY KEY NOT NULL, - title TEXT NOT NULL, - original_title TEXT, - release_year INTEGER NOT NULL, - media_type TEXT NOT NULL, - imdb_id TEXT NOT NULL UNIQUE, - imdb_score INTEGER, - imdb_votes INTEGER, - runtime INTEGER, - genres TEXT, - created TEXT NOT NULL, - updated TEXT NOT NULL -);; - -INSERT INTO _migrate_movies -SELECT - id, - title, - original_title, - release_year, - media_type, - imdb_id, - imdb_score, - imdb_votes, - runtime, - genres, - created, - updated -FROM movies -WHERE true;; - -DROP TABLE movies;; - -ALTER TABLE _migrate_movies -RENAME TO movies;;