diff --git a/metadex/metadex.py b/metadex/metadex.py
index 8f1783c..3ef5b81 100644
--- a/metadex/metadex.py
+++ b/metadex/metadex.py
@@ -202,7 +202,7 @@ def _scan_remove_missing(
                     if f.is_dir(follow_symlinks=False):
                         append(Path(f.path))
 
-                    expected.discard(f.name)
+                    expected.discard(Path(d["location"]).name)
 
             # `subdirs` sorts all changed dirs to the right, which means when we
             # extend `dirs` using `extendleft` it'll put them all left-most.
@@ -561,7 +561,7 @@ def _ingest_ls_remove_missing(
             elif action == "changed":
                 context.changed += 1
 
-            expected.discard(f.path.name)
+            expected.discard(Path(d["location"]).name)
 
         db.recalculate_dir_sizes(conn)