ducklet/unwind
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

remove transaction from imdb import

Browse source 
Removing the transaction will allow other (async) operations to succeed
while the import is running, otherwise the database will be locked.
It also actually speeds up the import process, and there's no good
reason to roll back if the import aborts mid run.
This commit is contained in:
ducklet 2021-07-11 17:23:19 +02:00
parent 294e311aff
commit 88fa1355cd
1 changed files with 24 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

50
unwind/imdb_import.py
View file

@ -189,40 +189,38 @@ async def import_from_file(*, basics_path: Path, ratings_path: Path):
perc = 0.0 perc = 0.0
perc_step = 0.001 perc_step = 0.001
async with db.shared_connection().transaction(): chunk = []
chunk = [] for i, m in enumerate(read_basics(basics_path)):
for i, m in enumerate(read_basics(basics_path)): if i / total > perc:
log.info("⏳ Imported %s%%", round(perc * 100, 1))
perc += perc_step
if i / total > perc: if m.media_type not in {
log.info("⏳ Imported %s%%", round(perc * 100, 1)) "Movie",
perc += perc_step "Short",
"TV Mini Series",
"TV Movie",
"TV Series",
"TV Short",
"TV Special",
"Video",
}:
log.debug("Skipping movie, unwanted media type: %s", m.media_type)
continue
if m.media_type not in { m.score = scores.get(m.imdb_id)
"Movie", chunk.append(m)
"Short",
"TV Mini Series",
"TV Movie",
"TV Series",
"TV Short",
"TV Special",
"Video",
}:
log.debug("Skipping movie, unwanted media type: %s", m.media_type)
continue
m.score = scores.get(m.imdb_id) if len(chunk) > 1000:
chunk.append(m)
if len(chunk) > 1000:
await add_or_update_many_movies(chunk)
chunk = []
if chunk:
await add_or_update_many_movies(chunk) await add_or_update_many_movies(chunk)
chunk = [] chunk = []
if chunk:
await add_or_update_many_movies(chunk)
chunk = []
async def load_from_web(): async def load_from_web():
"""Refresh the full IMDb movie database. """Refresh the full IMDb movie database.