From 88fa1355cd9e98e86ac95c2d12dc2a05541b1e7a Mon Sep 17 00:00:00 2001
From: ducklet <ducklet@noreply.code.dumpr.org>
Date: Sun, 11 Jul 2021 17:23:19 +0200
Subject: [PATCH] remove transaction from imdb import

Removing the transaction will allow other (async) operations to succeed
while the import is running, otherwise the database will be locked.
It also actually speeds up the import process, and there's no good
reason to roll back if the import aborts mid run.
---
 unwind/imdb_import.py | 50 +++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/unwind/imdb_import.py b/unwind/imdb_import.py
index 3cd139a..57d21a3 100644
--- a/unwind/imdb_import.py
+++ b/unwind/imdb_import.py
@@ -189,40 +189,38 @@ async def import_from_file(*, basics_path: Path, ratings_path: Path):
     perc = 0.0
     perc_step = 0.001
 
-    async with db.shared_connection().transaction():
+    chunk = []
 
-        chunk = []
+    for i, m in enumerate(read_basics(basics_path)):
 
-        for i, m in enumerate(read_basics(basics_path)):
+        if i / total > perc:
+            log.info("⏳ Imported %s%%", round(perc * 100, 1))
+            perc += perc_step
 
-            if i / total > perc:
-                log.info("⏳ Imported %s%%", round(perc * 100, 1))
-                perc += perc_step
+        if m.media_type not in {
+            "Movie",
+            "Short",
+            "TV Mini Series",
+            "TV Movie",
+            "TV Series",
+            "TV Short",
+            "TV Special",
+            "Video",
+        }:
+            log.debug("Skipping movie, unwanted media type: %s", m.media_type)
+            continue
 
-            if m.media_type not in {
-                "Movie",
-                "Short",
-                "TV Mini Series",
-                "TV Movie",
-                "TV Series",
-                "TV Short",
-                "TV Special",
-                "Video",
-            }:
-                log.debug("Skipping movie, unwanted media type: %s", m.media_type)
-                continue
+        m.score = scores.get(m.imdb_id)
+        chunk.append(m)
 
-            m.score = scores.get(m.imdb_id)
-            chunk.append(m)
-
-            if len(chunk) > 1000:
-                await add_or_update_many_movies(chunk)
-                chunk = []
-
-        if chunk:
+        if len(chunk) > 1000:
             await add_or_update_many_movies(chunk)
             chunk = []
 
+    if chunk:
+        await add_or_update_many_movies(chunk)
+        chunk = []
+
 
 async def load_from_web():
     """Refresh the full IMDb movie database.