diff --git a/CHANGES b/CHANGES
index fbeaf0763..f914c5a00 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,9 @@
 Release 1.0.7 (in development)
 ==============================
 
+* Fix a bug where the removal of download files and images wasn't
+  noticed.
+
 * #571: Implement ``~`` cross-reference prefix for the C domain.
 
 * Fix regression of LaTeX output with the fix of #556.
diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py
index 621c45aed..f4e08fbc4 100644
--- a/sphinx/util/__init__.py
+++ b/sphinx/util/__init__.py
@@ -114,9 +114,9 @@ class FilenameUniqDict(dict):
     def purge_doc(self, docname):
         for filename, (docs, _) in self.items():
             docs.discard(docname)
-            #if not docs:
-            #    del self[filename]
-            #    self._existing.discard(filename)
+            if not docs:
+                del self[filename]
+                self._existing.discard(filename)
 
     def __getstate__(self):
         return self._existing