ducklet/unwind
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

feat: use Alembic to initialize the database

Browse source 
This completely removes the previous DB patching mechanism.
When this is first run for an existing installation of Unwind, depending
on its version it might lead to problems because the database's schema
won't match the code.
To avoid that issue, when upgrading Unwind to this version make sure to
STOP the old application, install this new version but DON'T start it,
instead use `alembic upgrade head` to run the outstanding patches, and
only then start the application.
This commit is contained in:
ducklet 2024-05-19 02:25:36 +02:00
parent 5e4e70c9dc
commit 1ea09c1a45
20 changed files with 72 additions and 560 deletions
Download patch file Download diff file Expand all files Collapse all files

13
alembic/env.py
View file

@ -91,13 +91,18 @@ async def run_async_migrations() -> None:
await connectable.dispose()
def run_migrations_online_async() -> None:
def run_migrations_online() -> None:
"""Run migrations in 'online' mode."""
asyncio.run(run_async_migrations())
# Support having a (sync) connection passed in from another script.
if (conn := config.attributes.get("connection")) and isinstance(
conn, sa.Connection
):
do_run_migrations(conn)
else:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online_async()
run_migrations_online()

38
alembic/versions/1716077466-8b06e4916840_remove_db_patches_table.py Normal file
View file

@ -0,0 +1,38 @@
"""remove db_patches table
We replace our old patch process with Alembic's.
Revision ID: 8b06e4916840
Revises: f17c7ca9afa4
Create Date: 2024-05-19 00:11:06.730421+00:00
"""
from typing import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "8b06e4916840"
down_revision: str | None = "f17c7ca9afa4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("db_patches")
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"db_patches",
sa.Column("id", sa.INTEGER(), nullable=False),
sa.Column("current", sa.VARCHAR(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
# ### end Alembic commands ###

8
tests/test_db.py
View file

@ -20,14 +20,6 @@ def a_movie(**kwds) -> models.Movie:
return models.Movie(**args)
@pytest.mark.asyncio
async def test_current_patch_level(conn: db.Connection):
patch_level = "some-patch-level"
assert patch_level != await db.current_patch_level(conn)
await db.set_current_patch_level(conn, patch_level)
assert patch_level == await db.current_patch_level(conn)
@pytest.mark.asyncio
async def test_get(conn: db.Connection):
m1 = a_movie()

91
unwind/db.py
View file

@ -4,9 +4,12 @@ from pathlib import Path
from typing import Any, AsyncGenerator, Iterable, Literal, Sequence, Type, TypeVar
import sqlalchemy as sa
from sqlalchemy.dialects.sqlite import insert
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
import alembic.command
import alembic.config
import alembic.migration
from . import config
from .models import (
Model,
@ -15,7 +18,6 @@ from .models import (
Rating,
User,
asplain,
db_patches,
fromplain,
metadata,
movies,
@ -33,6 +35,25 @@ _engine: AsyncEngine | None = None
type Connection = AsyncConnection
_project_dir = Path(__file__).parent.parent
_alembic_ini = _project_dir / "alembic.ini"
def _init(conn: sa.Connection) -> None:
# See https://alembic.sqlalchemy.org/en/latest/cookbook.html#building-an-up-to-date-database-from-scratch
context = alembic.migration.MigrationContext.configure(conn)
heads = context.get_current_heads()
is_empty_db = not heads # We consider a DB empty if Alembic hasn't touched it yet.
if is_empty_db:
log.info("⚡️ Initializing empty database.")
metadata.create_all(conn)
# We pass our existing connection to Alembic's env.py, to avoid running another asyncio loop there.
alembic_cfg = alembic.config.Config(_alembic_ini)
alembic_cfg.attributes["connection"] = conn
alembic.command.stamp(alembic_cfg, "head")
async def open_connection_pool() -> None:
"""Open the DB connection pool.
@ -41,11 +62,7 @@ async def open_connection_pool() -> None:
"""
async with transaction() as conn:
await conn.execute(sa.text("PRAGMA journal_mode=WAL"))
await conn.run_sync(metadata.create_all, tables=[db_patches])
async with new_connection() as conn:
await apply_db_patches(conn)
await conn.run_sync(_init)
async def close_connection_pool() -> None:
@ -65,65 +82,7 @@ async def close_connection_pool() -> None:
await engine.dispose()
async def current_patch_level(conn: Connection, /) -> str:
query = sa.select(db_patches.c.current)
current = await conn.scalar(query)
return current or ""
async def set_current_patch_level(conn: Connection, /, current: str) -> None:
stmt = insert(db_patches).values(id=1, current=current)
stmt = stmt.on_conflict_do_update(set_={"current": stmt.excluded.current})
await conn.execute(stmt)
db_patches_dir = Path(__file__).parent / "sql"
async def apply_db_patches(conn: Connection, /) -> None:
"""Apply all remaining patches to the database.
Beware that patches will be applied in lexicographical order,
i.e. "10" comes before "9".
The current patch state is recorded in the DB itself.
Please note that every SQL statement in a patch file MUST be terminated
using two consecutive semi-colons (;).
Failing to do so will result in an error.
"""
applied_lvl = await current_patch_level(conn)
did_patch = False
for patchfile in sorted(db_patches_dir.glob("*.sql"), key=lambda p: p.stem):
patch_lvl = patchfile.stem
if patch_lvl <= applied_lvl:
continue
log.info("Applying patch: %s", patch_lvl)
sql = patchfile.read_text()
queries = sql.split(";;")
if len(queries) < 2:
log.error(
"Patch file is missing statement terminator (`;;'): %s", patchfile
)
raise RuntimeError("No statement found.")
async with transacted(conn):
for query in queries:
await conn.execute(sa.text(query))
await set_current_patch_level(conn, patch_lvl)
did_patch = True
if did_patch:
await _vacuum(conn)
async def _vacuum(conn: Connection, /) -> None:
async def vacuum(conn: Connection, /) -> None:
"""Vacuum the database.
This function cannot be run on a connection with an open transaction.

17
unwind/models.py
View file

@ -255,23 +255,6 @@ def utcnow() -> datetime:
return datetime.now(timezone.utc)
@mapper_registry.mapped
@dataclass
class DbPatch:
__table__: ClassVar[Table] = Table(
"db_patches",
metadata,
Column("id", Integer, primary_key=True),
Column("current", String),
)
id: int
current: str
db_patches = DbPatch.__table__
@mapper_registry.mapped
@dataclass
class Progress:

36
unwind/sql/00000000-init-0.sql
View file

@ -1,36 +0,0 @@
PRAGMA foreign_keys = ON;;
CREATE TABLE IF NOT EXISTS users (
id TEXT NOT NULL PRIMARY KEY,
imdb_id TEXT NOT NULL UNIQUE,
name TEXT NOT NULL
);;
CREATE TABLE IF NOT EXISTS movies (
id TEXT NOT NULL PRIMARY KEY,
title TEXT NOT NULL,
release_year NUMBER NOT NULL,
media_type TEXT NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
score NUMBER NOT NULL,
runtime NUMBER,
genres TEXT NOT NULL,
updated TEXT NOT NULL
);;
CREATE TABLE IF NOT EXISTS ratings (
id TEXT NOT NULL PRIMARY KEY,
movie_id TEXT NOT NULL,
user_id TEXT NOT NULL,
score NUMBER NOT NULL,
rating_date TEXT NOT NULL,
favorite NUMBER,
finished NUMBER,
FOREIGN KEY(movie_id) REFERENCES movies(id),
FOREIGN KEY(user_id) REFERENCES users(id)
);;
CREATE UNIQUE INDEX IF NOT EXISTS ratings_index ON ratings (
movie_id,
user_id
);;

40
unwind/sql/00000000-init-1.sql
View file

@ -1,40 +0,0 @@
-- add original_title to movies table
-- see https://www.sqlite.org/lang_altertable.html#caution
-- 1. Create new table
-- 2. Copy data
-- 3. Drop old table
-- 4. Rename new into old
CREATE TABLE _migrate_movies (
id TEXT NOT NULL PRIMARY KEY,
title TEXT NOT NULL,
original_title TEXT,
release_year NUMBER NOT NULL,
media_type TEXT NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
score NUMBER,
runtime NUMBER,
genres TEXT NOT NULL,
updated TEXT NOT NULL
);;
INSERT INTO _migrate_movies
SELECT
id,
title,
NULL,
release_year,
media_type,
imdb_id,
score,
runtime,
genres,
updated
FROM movies
WHERE true;;
DROP TABLE movies;;
ALTER TABLE _migrate_movies
RENAME TO movies;;

46
unwind/sql/00000000-init-2.sql
View file

@ -1,46 +0,0 @@
-- only set original_title if it differs from title,
-- and normalize media_type with an extra table.
CREATE TABLE mediatypes (
id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL UNIQUE
);;
INSERT INTO mediatypes (name)
SELECT DISTINCT media_type
FROM movies
WHERE true;;
CREATE TABLE _migrate_movies (
id TEXT PRIMARY KEY NOT NULL,
title TEXT NOT NULL,
original_title TEXT,
release_year INTEGER NOT NULL,
media_type_id INTEGER NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
score INTEGER,
runtime INTEGER,
genres TEXT NOT NULL,
updated TEXT NOT NULL,
FOREIGN KEY(media_type_id) REFERENCES mediatypes(id)
);;
INSERT INTO _migrate_movies
SELECT
id,
title,
(CASE WHEN original_title=title THEN NULL ELSE original_title END),
release_year,
(SELECT id FROM mediatypes WHERE name=media_type) AS media_type_id,
imdb_id,
score,
runtime,
genres,
updated
FROM movies
WHERE true;;
DROP TABLE movies;;
ALTER TABLE _migrate_movies
RENAME TO movies;;

62
unwind/sql/00000000-init-3.sql
View file

@ -1,62 +0,0 @@
-- add convenient view for movies
CREATE VIEW IF NOT EXISTS movies_view
AS SELECT
movies.id,
movies.title,
movies.original_title,
movies.release_year,
mediatypes.name AS media_type,
movies.imdb_id,
movies.score,
movies.runtime,
movies.genres,
movies.updated
FROM movies
JOIN mediatypes ON mediatypes.id=movies.media_type_id;;
CREATE TRIGGER IF NOT EXISTS insert_movies_view
INSTEAD OF INSERT
ON movies_view
BEGIN
INSERT INTO movies (
id,
title,
original_title,
release_year,
media_type_id,
imdb_id,
score,
runtime,
genres,
updated
) VALUES (
NEW.id,
NEW.title,
NEW.original_title,
NEW.release_year,
(SELECT id FROM mediatypes WHERE name=NEW.media_type),
NEW.imdb_id,
NEW.score,
NEW.runtime,
NEW.genres,
NEW.updated
);
END;;
CREATE TRIGGER IF NOT EXISTS update_movies_view
INSTEAD OF UPDATE OF media_type
ON movies_view
BEGIN
UPDATE movies
SET media_type_id=(SELECT id FROM mediatypes WHERE name=NEW.media_type)
WHERE id=OLD.id;
END;;
CREATE TRIGGER IF NOT EXISTS delete_movies_view
INSTEAD OF DELETE
ON movies_view
BEGIN
DELETE FROM movies
WHERE movies.id=OLD.id;
END;;

37
unwind/sql/00000000-init-4.sql
View file

@ -1,37 +0,0 @@
-- denormalize movie media_type
CREATE TABLE _migrate_movies (
id TEXT PRIMARY KEY NOT NULL,
title TEXT NOT NULL,
original_title TEXT,
release_year INTEGER NOT NULL,
media_type TEXT NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
score INTEGER,
runtime INTEGER,
genres TEXT NOT NULL,
updated TEXT NOT NULL
);;
INSERT INTO _migrate_movies
SELECT
id,
title,
original_title,
release_year,
(SELECT name FROM mediatypes WHERE id=media_type_id) AS media_type,
imdb_id,
score,
runtime,
genres,
updated
FROM movies
WHERE true;;
DROP VIEW movies_view;;
DROP TABLE mediatypes;;
DROP TABLE movies;;
ALTER TABLE _migrate_movies
RENAME TO movies;;

2
unwind/sql/00000001-fix-db.sql.disabled
View file

@ -1,2 +0,0 @@
-- see the commit of this file for details.
;;

8
unwind/sql/20210705-224139.sql
View file

@ -1,8 +0,0 @@
-- add groups table
CREATE TABLE groups (
id TEXT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
secret TEXT NOT NULL,
users TEXT NOT NULL -- JSON array
);;

7
unwind/sql/20210711-172808--progress-table.sql
View file

@ -1,7 +0,0 @@
-- add progress table
CREATE TABLE progress (
id TEXT PRIMARY KEY NOT NULL,
state TEXT NOT NULL,
started TEXT NOT NULL
);;

36
unwind/sql/20210720-213416.sql
View file

@ -1,36 +0,0 @@
-- add IMDb vote count
CREATE TABLE _migrate_movies (
id TEXT PRIMARY KEY NOT NULL,
title TEXT NOT NULL,
original_title TEXT,
release_year INTEGER NOT NULL,
media_type TEXT NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
imdb_score INTEGER,
imdb_votes INTEGER,
runtime INTEGER,
genres TEXT NOT NULL,
updated TEXT NOT NULL
);;
INSERT INTO _migrate_movies
SELECT
id,
title,
original_title,
release_year,
media_type,
imdb_id,
score AS imdb_score,
NULL AS imdb_votes,
runtime,
genres,
updated
FROM movies
WHERE true;;
DROP TABLE movies;;
ALTER TABLE _migrate_movies
RENAME TO movies;;

24
unwind/sql/20210720-223416.sql
View file

@ -1,24 +0,0 @@
-- add IMDb vote count
CREATE TABLE _migrate_progress (
id TEXT PRIMARY KEY NOT NULL,
type TEXT NOT NULL,
state TEXT NOT NULL,
started TEXT NOT NULL,
stopped TEXT
);;
INSERT INTO _migrate_progress
SELECT
id,
'import-imdb-movies' AS type,
state,
started,
NULL AS stopped
FROM progress
WHERE true;;
DROP TABLE progress;;
ALTER TABLE _migrate_progress
RENAME TO progress;;

38
unwind/sql/20210721-213417.sql
View file

@ -1,38 +0,0 @@
-- add creation timestamp to movies
CREATE TABLE _migrate_movies (
id TEXT PRIMARY KEY NOT NULL,
title TEXT NOT NULL,
original_title TEXT,
release_year INTEGER NOT NULL,
media_type TEXT NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
imdb_score INTEGER,
imdb_votes INTEGER,
runtime INTEGER,
genres TEXT NOT NULL,
created TEXT NOT NULL,
updated TEXT NOT NULL
);;
INSERT INTO _migrate_movies
SELECT
id,
title,
original_title,
release_year,
media_type,
imdb_id,
imdb_score,
imdb_votes,
runtime,
genres,
updated AS created,
updated
FROM movies
WHERE true;;
DROP TABLE movies;;
ALTER TABLE _migrate_movies
RENAME TO movies;;

24
unwind/sql/20210728-223416.sql
View file

@ -1,24 +0,0 @@
-- add IMDb vote count
CREATE TABLE _migrate_progress (
id TEXT PRIMARY KEY NOT NULL,
type TEXT NOT NULL,
state TEXT NOT NULL,
started TEXT NOT NULL,
stopped TEXT
);;
INSERT INTO _migrate_progress
SELECT
id,
type,
'{"percent":' || state || '}' AS state,
started,
stopped
FROM progress
WHERE true;;
DROP TABLE progress;;
ALTER TABLE _migrate_progress
RENAME TO progress;;

22
unwind/sql/20210801-201151--add-user-secret.sql
View file

@ -1,22 +0,0 @@
-- add secret to users
CREATE TABLE _migrate_users (
id TEXT PRIMARY KEY NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
secret TEXT NOT NULL
);;
INSERT INTO _migrate_users
SELECT
id,
imdb_id,
name,
'' AS secret
FROM users
WHERE true;;
DROP TABLE users;;
ALTER TABLE _migrate_users
RENAME TO users;;

45
unwind/sql/20210802-212312--add-group-admins.sql
View file

@ -1,45 +0,0 @@
-- add group admins
--- remove secrets from groups
CREATE TABLE _migrate_groups (
id TEXT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
users TEXT NOT NULL -- JSON array
);;
INSERT INTO _migrate_groups
SELECT
id,
name,
users
FROM groups
WHERE true;;
DROP TABLE groups;;
ALTER TABLE _migrate_groups
RENAME TO groups;;
--- add group access to users
CREATE TABLE _migrate_users (
id TEXT PRIMARY KEY NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
secret TEXT NOT NULL,
groups TEXT NOT NULL -- JSON array
);;
INSERT INTO _migrate_users
SELECT
id,
imdb_id,
name,
secret,
'[]' AS groups
FROM users
WHERE true;;
DROP TABLE users;;
ALTER TABLE _migrate_users
RENAME TO users;;

38
unwind/sql/20240511-001949--remove-genres-notnull.sql
View file

@ -1,38 +0,0 @@
-- remove NOTNULL constraint from movies.genres
CREATE TABLE _migrate_movies (
id TEXT PRIMARY KEY NOT NULL,
title TEXT NOT NULL,
original_title TEXT,
release_year INTEGER NOT NULL,
media_type TEXT NOT NULL,
imdb_id TEXT NOT NULL UNIQUE,
imdb_score INTEGER,
imdb_votes INTEGER,
runtime INTEGER,
genres TEXT,
created TEXT NOT NULL,
updated TEXT NOT NULL
);;
INSERT INTO _migrate_movies
SELECT
id,
title,
original_title,
release_year,
media_type,
imdb_id,
imdb_score,
imdb_votes,
runtime,
genres,
created,
updated
FROM movies
WHERE true;;
DROP TABLE movies;;
ALTER TABLE _migrate_movies
RENAME TO movies;;