diff --git a/sphinx/environment.py b/sphinx/environment.py
index 525b999ac..85e832ccc 100644
--- a/sphinx/environment.py
+++ b/sphinx/environment.py
@@ -1444,7 +1444,7 @@ class BuildEnvironment:
         # a list of all docnames whose section numbers changed
         rewrite_needed = []
 
-        assigned = []
+        assigned = set()
         old_secnumbers = self.toc_secnumbers
         self.toc_secnumbers = {}
 
@@ -1489,14 +1489,14 @@ class BuildEnvironment:
                     continue
                 if ref in self.tocs:
                     secnums = self.toc_secnumbers[ref] = {}
-                    assigned.append(ref)
+                    assigned.add(ref)
                     _walk_toc(self.tocs[ref], secnums, depth,
                               self.titles.get(ref))
                     if secnums != old_secnumbers.get(ref):
                         rewrite_needed.append(ref)
 
         for docname in self.numbered_toctrees:
-            assigned.append(docname)
+            assigned.add(docname)
             doctree = self.get_doctree(docname)
             for toctreenode in doctree.traverse(addnodes.toctree):
                 depth = toctreenode.get('numbered', 0)