Fix #3633: misdetect unreferenced citations

CHANGES

@@ -5,6 +5,7 @@ Incompatible changes
 --------------------
 
 * #3345: Replace the custom smartypants code with docutils' smart_quotes
+* Add line numbers to citation data in std domain
 
 Deprecated
 ----------
@@ -15,12 +16,16 @@ Deprecated
 Features added
 --------------
 
+* Add a new event `env-check-consistency` to check consistency to extensions
+* Add `Domain.check_consistency()` to check consistency
+
 Bugs fixed
 ----------
 
 * #3661: sphinx-build crashes on parallel build
 * #3669: gettext builder fails with "ValueError: substring not found"
 * #3660: Sphinx always depends on sphinxcontrib-websupport and its dependencies
+* #3633: misdetect unreferenced citations
 
 Testing
 --------

doc/extdev/appapi.rst

@@ -590,6 +590,13 @@ handlers to the events.  Example:
    .. versionchanged:: 1.3
       The handlers' return value is now used.
 
+.. event:: env-check-consistency (env)
+
+   Emmited when Consistency checks phase.  You can check consistency of
+   metadata for whole of documents.
+
+   .. versionadded:: 1.6
+
 .. event:: html-collect-pages (app)
 
    Emitted when the HTML builder is starting to write non-document pages.  You

sphinx/domains/__init__.py

@@ -240,6 +240,11 @@ class Domain(object):
         """Process a document after it is read by the environment."""
         pass
 
+    def check_consistency(self):
+        # type: () -> None
+        """Do consistency checks."""
+        pass
+
     def process_field_xref(self, pnode):
         # type: (nodes.Node) -> None
         """Process a pending xref created in a doc field.

sphinx/domains/std.py

@@ -492,7 +492,8 @@ class StandardDomain(Domain):
     initial_data = {
         'progoptions': {},      # (program, name) -> docname, labelid
         'objects': {},          # (type, name) -> docname, labelid
-        'citations': {},    # name -> docname, labelid
+        'citations': {},        # name -> docname, labelid, lineno
+        'citation_refs': {},    # name -> list of docnames
         'labels': {             # labelname -> docname, labelid, sectionname
             'genindex': ('genindex', '', l_('Index')),
             'modindex': ('py-modindex', '', l_('Module Index')),
@@ -530,9 +531,14 @@ class StandardDomain(Domain):
         for key, (fn, _l) in list(self.data['objects'].items()):
             if fn == docname:
                 del self.data['objects'][key]
-        for key, (fn, _l) in list(self.data['citations'].items()):
+        for key, (fn, _l, lineno) in list(self.data['citations'].items()):
             if fn == docname:
                 del self.data['citations'][key]
+        for key, docnames in list(self.data['citation_refs'].items()):
+            if docnames == [docname]:
+                del self.data['citation_refs'][key]
+            elif docname in docnames:
+                docnames.pop(docname)
         for key, (fn, _l, _l) in list(self.data['labels'].items()):
             if fn == docname:
                 del self.data['labels'][key]
@@ -552,6 +558,11 @@ class StandardDomain(Domain):
         for key, data in otherdata['citations'].items():
             if data[0] in docnames:
                 self.data['citations'][key] = data
+        for key, data in otherdata['citation_refs'].items():
+            citation_refs = self.data['citation_refs'].setdefault(key, [])
+            for docname in data:
+                if docname in docnames:
+                    citation_refs.append(docname)
         for key, data in otherdata['labels'].items():
             if data[0] in docnames:
                 self.data['labels'][key] = data
@@ -562,6 +573,7 @@ class StandardDomain(Domain):
     def process_doc(self, env, docname, document):
         # type: (BuildEnvironment, unicode, nodes.Node) -> None
         self.note_citations(env, docname, document)
+        self.note_citation_refs(env, docname, document)
         self.note_labels(env, docname, document)
 
     def note_citations(self, env, docname, document):
@@ -572,7 +584,13 @@ class StandardDomain(Domain):
                 path = env.doc2path(self.data['citations'][label][0])
                 logger.warning('duplicate citation %s, other instance in %s', label, path,
                                location=node, type='ref', subtype='citation')
-            self.data['citations'][label] = (docname, node['ids'][0])
+            self.data['citations'][label] = (docname, node['ids'][0], node.line)
+
+    def note_citation_refs(self, env, docname, document):
+        # type: (BuildEnvironment, unicode, nodes.Node) -> None
+        for name, refs in iteritems(document.citation_refs):
+            citation_refs = self.data['citation_refs'].setdefault(name, [])
+            citation_refs.append(docname)
 
     def note_labels(self, env, docname, document):
         # type: (BuildEnvironment, unicode, nodes.Node) -> None
@@ -614,6 +632,14 @@ class StandardDomain(Domain):
                 continue
             labels[name] = docname, labelid, sectname
 
+    def check_consistency(self):
+        # type: () -> None
+        for name, (docname, labelid, lineno) in iteritems(self.data['citations']):
+            if labelid not in self.data['citation_refs']:
+                logger.warning('Citation [%s] is not referenced.', name,
+                               type='ref', subtype='citation',
+                               location=(docname, lineno))
+
     def build_reference_node(self, fromdocname, builder, docname, labelid,
                              sectname, rolename, **options):
         # type: (unicode, Builder, unicode, unicode, unicode, unicode, Any) -> nodes.Node
@@ -788,7 +814,7 @@ class StandardDomain(Domain):
         # type: (BuildEnvironment, unicode, Builder, unicode, unicode, nodes.Node, nodes.Node) -> nodes.Node  # NOQA
         from sphinx.environment import NoUri
 
-        docname, labelid = self.data['citations'].get(target, ('', ''))
+        docname, labelid, lineno = self.data['citations'].get(target, ('', '', 0))
         if not docname:
             if 'ids' in node:
                 # remove ids attribute that annotated at

sphinx/environment/__init__.py

@@ -78,7 +78,7 @@ default_settings = {
 # or changed to properly invalidate pickle files.
 #
 # NOTE: increase base version by 2 to have distinct numbers for Py2 and 3
-ENV_VERSION = 51 + (sys.version_info[0] - 2)
+ENV_VERSION = 52 + (sys.version_info[0] - 2)
 
 
 dummy_reporter = Reporter('', 4, 4)
@@ -1014,3 +1014,8 @@ class BuildEnvironment(object):
                     continue
                 logger.warning('document isn\'t included in any toctree',
                                location=docname)
+
+        # call check-consistency for all extensions
+        for domain in self.domains.values():
+            domain.check_consistency()
+        self.app.emit('env-check-consistency', self)

sphinx/events.py

@@ -31,6 +31,7 @@ core_events = {
     'env-get-updated': 'env',
     'env-purge-doc': 'env, docname',
     'env-before-read-docs': 'env, docnames',
+    'env-check-consistency': 'env',
     'source-read': 'docname, source text',
     'doctree-read': 'the doctree before being pickled',
     'env-merge-info': 'env, read docnames, other env instance',

sphinx/transforms/__init__.py

@@ -296,12 +296,6 @@ class UnreferencedFootnotesDetector(SphinxTransform):
                                type='ref', subtype='footnote',
                                location=node)
 
-        for node in self.document.citations:
-            if node['names'][0] not in self.document.citation_refs:
-                logger.warning('Citation [%s] is not referenced.', node['names'][0],
-                               type='ref', subtype='citation',
-                               location=node)
-
 
 class FilterSystemMessages(SphinxTransform):
     """Filter system messages from a doctree."""