diff --git a/metadex/metadex.py b/metadex/metadex.py
index eed2db6..8f1783c 100644
--- a/metadex/metadex.py
+++ b/metadex/metadex.py
@@ -387,10 +387,33 @@ def ingest_rclone_json(
 ) -> _LogContext:
     is_ignored = ignore.parse(ignore_file)
 
+    expected: "set[str]" = set()
+    parent: "Path | None" = None
+
     context = _LogContext()
 
     with db.transaction() as conn:
         for d in _parse_rclone_json(file, remote_base=remote_base):
+            if remove_missing:
+                new_parent = Path(d["location"]).parent
+                if parent != new_parent:  # Dir changed
+                    if parent is None:
+                        assert not expected
+                    else:
+                        # remove missing
+                        for name in expected:
+                            loc = str(parent / name)
+                            if is_ignored(loc):
+                                log.info("Ignoring file (for removal): %a", loc)
+                                continue
+
+                            log.info("File removed: %a", loc)
+
+                            context.removed += db.remove_all(conn, loc)
+
+                    parent = new_parent
+
+                    expected = {name for name in db.files_in_dir(conn, str(new_parent))}
 
             context.seen += 1
 
@@ -406,6 +429,21 @@ def ingest_rclone_json(
             elif action == "changed":
                 context.changed += 1
 
+            if remove_missing:
+                expected.discard(Path(d["location"]).name)
+
+        if remove_missing:
+            if parent is not None:
+                for name in expected:
+                    loc = str(parent / name)
+                    if is_ignored(loc):
+                        log.info("Ignoring file (for removal): %a", loc)
+                        continue
+
+                    log.info("File removed: %a", loc)
+
+                    context.removed += db.remove_all(conn, loc)
+
         db.recalculate_dir_sizes(conn)
 
     return context