ducklet/metadex
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

support remove_missing in ingest_rclone_json

Browse source
This commit is contained in:
ducklet 2023-02-11 18:48:30 +01:00
parent 19deeb9512
commit d8afd3a293
1 changed files with 38 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

38
metadex/metadex.py
View file

@ -387,10 +387,33 @@ def ingest_rclone_json(
) -> _LogContext:
is_ignored = ignore.parse(ignore_file)
expected: "set[str]" = set()
parent: "Path | None" = None
context = _LogContext()
with db.transaction() as conn:
for d in _parse_rclone_json(file, remote_base=remote_base):
if remove_missing:
new_parent = Path(d["location"]).parent
if parent != new_parent: # Dir changed
if parent is None:
assert not expected
else:
# remove missing
for name in expected:
loc = str(parent / name)
if is_ignored(loc):
log.info("Ignoring file (for removal): %a", loc)
continue
log.info("File removed: %a", loc)
context.removed += db.remove_all(conn, loc)
parent = new_parent
expected = {name for name in db.files_in_dir(conn, str(new_parent))}
context.seen += 1
@ -406,6 +429,21 @@ def ingest_rclone_json(
elif action == "changed":
context.changed += 1
if remove_missing:
expected.discard(Path(d["location"]).name)
if remove_missing:
if parent is not None:
for name in expected:
loc = str(parent / name)
if is_ignored(loc):
log.info("Ignoring file (for removal): %a", loc)
continue
log.info("File removed: %a", loc)
context.removed += db.remove_all(conn, loc)
db.recalculate_dir_sizes(conn)
return context