Further improve type annotations, reduce mypy whitelist (#10770)

Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com>

setup.cfg

@@ -47,22 +47,25 @@ strict_optional = False
 [mypy-sphinx.builders.html]
 strict_optional = False
 
-[mypy-sphinx.builders.latex.*]
+[mypy-sphinx.builders.latex]
 strict_optional = False
 
 [mypy-sphinx.builders.linkcheck]
 strict_optional = False
 
-[mypy-sphinx.cmd.*]
-strict_optional = False
-
 [mypy-sphinx.directives.code]
 strict_optional = False
 
 [mypy-sphinx.domains.*]
 strict_optional = False
 
-[mypy-sphinx.environment.*]
+[mypy-sphinx.environment]
+strict_optional = False
+
+[mypy-sphinx.environment.adapters.toctree]
+strict_optional = False
+
+[mypy-sphinx.environment.adapters.indexentries]
 strict_optional = False
 
 [mypy-sphinx.ext.*]
@@ -83,13 +86,28 @@ strict_optional = False
 [mypy-sphinx.testing.util]
 strict_optional = False
 
-[mypy-sphinx.transforms.*]
+[mypy-sphinx.transforms.i18n]
+strict_optional = False
+
+[mypy-sphinx.transforms.post_transforms.images]
 strict_optional = False
 
 [mypy-sphinx.util.*]
 strict_optional = False
 
-[mypy-sphinx.writers.*]
+[mypy-sphinx.writers.html]
+strict_optional = False
+
+[mypy-sphinx.writers.html5]
+strict_optional = False
+
+[mypy-sphinx.writers.latex]
+strict_optional = False
+
+[mypy-sphinx.writers.text]
+strict_optional = False
+
+[mypy-sphinx.writers.xml]
 strict_optional = False
 
 [tool:pytest]

sphinx/builders/latex/theming.py

@@ -2,7 +2,7 @@
 
 import configparser
 from os import path
-from typing import Dict
+from typing import Dict, Optional
 
 from sphinx.application import Sphinx
 from sphinx.config import Config
@@ -113,14 +113,12 @@ class ThemeFactory:
         if name in self.themes:
             theme = self.themes[name]
         else:
-            theme = self.find_user_theme(name)
-            if not theme:
-                theme = Theme(name)
+            theme = self.find_user_theme(name) or Theme(name)
 
         theme.update(self.config)
         return theme
 
-    def find_user_theme(self, name: str) -> Theme:
+    def find_user_theme(self, name: str) -> Optional[Theme]:
         """Find a theme named as *name* from latex_theme_path."""
         for theme_path in self.theme_paths:
             config_path = path.join(theme_path, name, 'theme.conf')

sphinx/builders/latex/transforms.py

@@ -1,6 +1,6 @@
 """Transforms for LaTeX builder."""
 
-from typing import Any, Dict, List, Set, Tuple, cast
+from typing import Any, Dict, List, Optional, Set, Tuple, cast
 
 from docutils import nodes
 from docutils.nodes import Element, Node
@@ -94,13 +94,17 @@ class ShowUrlsTransform(SphinxPostTransform):
     def get_docname_for_node(self, node: Node) -> str:
         while node:
             if isinstance(node, nodes.document):
-                return self.env.path2doc(node['source'])
+                return self.env.path2doc(node['source']) or ''
             elif isinstance(node, addnodes.start_of_file):
                 return node['docname']
             else:
                 node = node.parent
 
-        return None  # never reached here. only for type hinting
+        try:
+            source = node['source']  # type: ignore[index]
+        except TypeError:
+            raise ValueError('Failed to get a docname!') from None
+        raise ValueError(f'Failed to get a docname for source {source!r}!')
 
     def create_footnote(self, uri: str, docname: str) -> Tuple[nodes.footnote, nodes.footnote_reference]:  # NOQA
         reference = nodes.reference('', nodes.Text(uri), refuri=uri, nolinkurl=True)
@@ -355,7 +359,7 @@ class LaTeXFootnoteVisitor(nodes.NodeVisitor):
         self.footnotes: List[nodes.footnote] = footnotes
         self.pendings: List[nodes.footnote] = []
         self.table_footnotes: List[nodes.footnote] = []
-        self.restricted: Element = None
+        self.restricted: Optional[Element] = None
         super().__init__(document)
 
     def unknown_visit(self, node: Node) -> None:
@@ -433,7 +437,7 @@ class LaTeXFootnoteVisitor(nodes.NodeVisitor):
         number = node.astext().strip()
         docname = node['docname']
         if (docname, number) in self.appeared:
-            footnote = self.appeared.get((docname, number))
+            footnote = self.appeared[(docname, number)]
             footnote["referred"] = True
 
             mark = footnotemark('', number, refid=node['refid'])
@@ -458,7 +462,7 @@ class LaTeXFootnoteVisitor(nodes.NodeVisitor):
             if docname == footnote['docname'] and footnote['ids'][0] == node['refid']:
                 return footnote
 
-        return None
+        raise ValueError('No footnote not found for given reference node %r' % node)
 
 
 class BibliographyTransform(SphinxPostTransform):

sphinx/cmd/build.py

@@ -9,7 +9,7 @@ import pdb
 import sys
 import traceback
 from os import path
-from typing import IO, Any, List
+from typing import IO, Any, List, Optional, TextIO
 
 from docutils.utils import SystemMessage
 
@@ -24,7 +24,9 @@ from sphinx.util.docutils import docutils_namespace, patch_docutils
 from sphinx.util.osutil import abspath, ensuredir
 
 
-def handle_exception(app: Sphinx, args: Any, exception: BaseException, stderr: IO = sys.stderr) -> None:  # NOQA
+def handle_exception(
+    app: Optional[Sphinx], args: Any, exception: BaseException, stderr: IO = sys.stderr
+) -> None:
     if isinstance(exception, bdb.BdbQuit):
         return
 
@@ -222,8 +224,8 @@ def build_main(argv: List[str] = sys.argv[1:]) -> int:
     if args.color == 'no' or (args.color == 'auto' and not color_terminal()):
         nocolor()
 
-    status = sys.stdout
-    warning = sys.stderr
+    status: Optional[TextIO] = sys.stdout
+    warning: Optional[TextIO] = sys.stderr
     error = sys.stderr
 
     if args.quiet:

sphinx/cmd/make_mode.py

@@ -11,7 +11,7 @@ import os
 import subprocess
 import sys
 from os import path
-from typing import List
+from typing import List, Optional
 
 import sphinx
 from sphinx.cmd.build import build_main
@@ -132,7 +132,7 @@ class Make:
             return 1
         return 0
 
-    def run_generic_build(self, builder: str, doctreedir: str = None) -> int:
+    def run_generic_build(self, builder: str, doctreedir: Optional[str] = None) -> int:
         # compatibility with old Makefile
         papersize = os.getenv('PAPER', '')
         opts = self.opts

sphinx/cmd/quickstart.py

@@ -7,14 +7,14 @@ import sys
 import time
 from collections import OrderedDict
 from os import path
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 
 # try to import readline, unix specific enhancement
 try:
     import readline
     if TYPE_CHECKING and sys.platform == "win32":  # always false, for type checking
         raise ImportError
-
+    READLINE_AVAILABLE = True
     if readline.__doc__ and 'libedit' in readline.__doc__:
         readline.parse_and_bind("bind ^I rl_complete")
         USE_LIBEDIT = True
@@ -22,7 +22,7 @@ try:
         readline.parse_and_bind("tab: complete")
         USE_LIBEDIT = False
 except ImportError:
-    readline = None
+    READLINE_AVAILABLE = False
     USE_LIBEDIT = False
 
 from docutils.utils import column_width
@@ -130,7 +130,9 @@ def ok(x: str) -> str:
     return x
 
 
-def do_prompt(text: str, default: str = None, validator: Callable[[str], Any] = nonempty) -> Union[str, bool]:  # NOQA
+def do_prompt(
+    text: str, default: Optional[str] = None, validator: Callable[[str], Any] = nonempty
+) -> Union[str, bool]:
     while True:
         if default is not None:
             prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default)
@@ -141,7 +143,7 @@ def do_prompt(text: str, default: str = None, validator: Callable[[str], Any] =
             # sequence (see #5335).  To avoid the problem, all prompts are not colored
             # on libedit.
             pass
-        elif readline:
+        elif READLINE_AVAILABLE:
             # pass input_mode=True if readline available
             prompt = colorize(COLOR_QUESTION, prompt, input_mode=True)
         else:
@@ -159,7 +161,7 @@ def do_prompt(text: str, default: str = None, validator: Callable[[str], Any] =
 
 
 class QuickstartRenderer(SphinxRenderer):
-    def __init__(self, templatedir: str) -> None:
+    def __init__(self, templatedir: Optional[str]) -> None:
         self.templatedir = templatedir or ''
         super().__init__()
 
@@ -321,8 +323,9 @@ def ask_user(d: Dict[str, Any]) -> None:
     print()
 
 
-def generate(d: Dict, overwrite: bool = True, silent: bool = False, templatedir: str = None
-             ) -> None:
+def generate(
+    d: Dict, overwrite: bool = True, silent: bool = False, templatedir: Optional[str] = None
+) -> None:
     """Generate project based on values in *d*."""
     template = QuickstartRenderer(templatedir=templatedir)
 
@@ -357,7 +360,7 @@ def generate(d: Dict, overwrite: bool = True, silent: bool = False, templatedir:
     ensuredir(path.join(srcdir, d['dot'] + 'templates'))
     ensuredir(path.join(srcdir, d['dot'] + 'static'))
 
-    def write_file(fpath: str, content: str, newline: str = None) -> None:
+    def write_file(fpath: str, content: str, newline: Optional[str] = None) -> None:
         if overwrite or not path.isfile(fpath):
             if 'quiet' not in d:
                 print(__('Creating file %s.') % fpath)

sphinx/environment/__init__.py

@@ -91,7 +91,7 @@ class BuildEnvironment:
 
     # --------- ENVIRONMENT INITIALIZATION -------------------------------------
 
-    def __init__(self, app: "Sphinx" = None):
+    def __init__(self, app: Optional["Sphinx"] = None):
         self.app: Sphinx = None
         self.doctreedir: str = None
         self.srcdir: str = None
@@ -327,7 +327,7 @@ class BuildEnvironment:
         """
         return self.project.doc2path(docname, base)
 
-    def relfn2path(self, filename: str, docname: str = None) -> Tuple[str, str]:
+    def relfn2path(self, filename: str, docname: Optional[str] = None) -> Tuple[str, str]:
         """Return paths to a file referenced from a document, relative to
         documentation root and absolute.
 
@@ -340,7 +340,7 @@ class BuildEnvironment:
             rel_fn = filename[1:]
         else:
             docdir = path.dirname(self.doc2path(docname or self.docname,
-                                                base=None))
+                                                base=False))
             rel_fn = path.join(docdir, filename)
 
         return (canon_path(path.normpath(rel_fn)),
@@ -488,7 +488,9 @@ class BuildEnvironment:
 
         *filename* should be absolute or relative to the source directory.
         """
-        self.included[self.docname].add(self.path2doc(filename))
+        doc = self.path2doc(filename)
+        if doc:
+            self.included[self.docname].add(doc)
 
     def note_reread(self) -> None:
         """Add the current document to the list of documents that will
@@ -517,9 +519,14 @@ class BuildEnvironment:
         doctree.reporter = LoggingReporter(self.doc2path(docname))
         return doctree
 
-    def get_and_resolve_doctree(self, docname: str, builder: "Builder",
-                                doctree: nodes.document = None, prune_toctrees: bool = True,
-                                includehidden: bool = False) -> nodes.document:
+    def get_and_resolve_doctree(
+        self,
+        docname: str,
+        builder: "Builder",
+        doctree: Optional[nodes.document] = None,
+        prune_toctrees: bool = True,
+        includehidden: bool = False
+    ) -> nodes.document:
         """Read the doctree from the pickle, resolve cross-references and
         toctrees and return it.
         """
@@ -543,7 +550,7 @@ class BuildEnvironment:
 
     def resolve_toctree(self, docname: str, builder: "Builder", toctree: addnodes.toctree,
                         prune: bool = True, maxdepth: int = 0, titles_only: bool = False,
-                        collapse: bool = False, includehidden: bool = False) -> Node:
+                        collapse: bool = False, includehidden: bool = False) -> Optional[Node]:
         """Resolve a *toctree* node into individual bullet lists with titles
         as items, returning None (if no containing titles are found) or
         a new node.
@@ -583,7 +590,9 @@ class BuildEnvironment:
     def collect_relations(self) -> Dict[str, List[Optional[str]]]:
         traversed = set()
 
-        def traverse_toctree(parent: str, docname: str) -> Iterator[Tuple[str, str]]:
+        def traverse_toctree(
+            parent: Optional[str], docname: str
+        ) -> Iterator[Tuple[Optional[str], str]]:
             if parent == docname:
                 logger.warning(__('self referenced toctree found. Ignored.'),
                                location=docname, type='toc',

sphinx/environment/adapters/indexentries.py

@@ -25,7 +25,7 @@ class IndexEntries:
         """Create the real index from the collected index entries."""
         new: Dict[str, List] = {}
 
-        def add_entry(word: str, subword: str, main: str, link: bool = True,
+        def add_entry(word: str, subword: str, main: Optional[str], link: bool = True,
                       dic: Dict[str, List] = new, key: Optional[str] = None) -> None:
             # Force the word to be unicode if it's a ASCII bytestring.
             # This will solve problems with unicode normalization later.

sphinx/environment/collectors/toctree.py

@@ -1,6 +1,6 @@
 """Toctree collector for sphinx.environment."""
 
-from typing import Any, Dict, List, Set, Tuple, Type, TypeVar, cast
+from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, cast
 
 from docutils import nodes
 from docutils.nodes import Element, Node
@@ -66,7 +66,7 @@ class TocTreeCollector(EnvironmentCollector):
                     result.extend(traverse_in_section(child, cls))
             return result
 
-        def build_toc(node: Element, depth: int = 1) -> nodes.bullet_list:
+        def build_toc(node: Element, depth: int = 1) -> Optional[nodes.bullet_list]:
             entries: List[Element] = []
             for sectionnode in node:
                 # find all toctree nodes in this section and add them
@@ -132,7 +132,9 @@ class TocTreeCollector(EnvironmentCollector):
         old_secnumbers = env.toc_secnumbers
         env.toc_secnumbers = {}
 
-        def _walk_toc(node: Element, secnums: Dict, depth: int, titlenode: nodes.title = None) -> None:  # NOQA
+        def _walk_toc(
+            node: Element, secnums: Dict, depth: int, titlenode: Optional[nodes.title] = None
+        ) -> None:
             # titlenode is the title of the document, it will get assigned a
             # secnumber too, so that it shows up in next/prev/parent rellinks
             for subnode in node.children:
@@ -207,7 +209,7 @@ class TocTreeCollector(EnvironmentCollector):
         env.toc_fignumbers = {}
         fignum_counter: Dict[str, Dict[Tuple[int, ...], int]] = {}
 
-        def get_figtype(node: Node) -> str:
+        def get_figtype(node: Node) -> Optional[str]:
             for domain in env.domains.values():
                 figtype = domain.get_enumerable_node_type(node)
                 if domain.name == 'std' and not domain.get_numfig_title(node):  # type: ignore

sphinx/transforms/post_transforms/__init__.py

@@ -112,7 +112,9 @@ class ReferencesResolver(SphinxPostTransform):
 
             node.replace_self(newnodes)
 
-    def resolve_anyref(self, refdoc: str, node: pending_xref, contnode: Element) -> Element:
+    def resolve_anyref(
+        self, refdoc: str, node: pending_xref, contnode: Element
+    ) -> Optional[Element]:
         """Resolve reference generated by the "any" role."""
         stddomain = self.env.get_domain('std')
         target = node['reftarget']

sphinx/util/__init__.py

@@ -212,7 +212,7 @@ _DEBUG_HEADER = '''\
 '''
 
 
-def save_traceback(app: "Sphinx") -> str:
+def save_traceback(app: Optional["Sphinx"]) -> str:
     """Save the current exception's traceback in a temporary file."""
     import platform
 

sphinx/writers/texinfo.py

@@ -115,7 +115,7 @@ class TexinfoWriter(writers.Writer):
 
     settings_defaults: Dict = {}
 
-    output: str = None
+    output: Optional[str] = None  # type: ignore[assignment]
 
     visitor_attributes = ('output', 'fragment')
 
@@ -134,8 +134,8 @@ class TexinfoWriter(writers.Writer):
 
 class TexinfoTranslator(SphinxTranslator):
 
-    builder: "TexinfoBuilder" = None
     ignore_missing_images = False
+    builder: "TexinfoBuilder"
 
     default_elements = {
         'author': '',
@@ -184,7 +184,7 @@ class TexinfoTranslator(SphinxTranslator):
         self.in_footnote = 0
         self.in_samp = 0
         self.handled_abbrs: Set[str] = set()
-        self.colwidths: Optional[List[int]] = None
+        self.colwidths: List[int] = []
 
     def finish(self) -> None:
         if self.previous_section is None:
@@ -765,10 +765,10 @@ class TexinfoTranslator(SphinxTranslator):
         self.ensure_eol()
         self.body.append('@end quotation\n')
 
-    def visit_literal_block(self, node: Element) -> None:
+    def visit_literal_block(self, node: Optional[Element]) -> None:
         self.body.append('\n@example\n')
 
-    def depart_literal_block(self, node: Element) -> None:
+    def depart_literal_block(self, node: Optional[Element]) -> None:
         self.ensure_eol()
         self.body.append('@end example\n')
 
@@ -1404,7 +1404,7 @@ class TexinfoTranslator(SphinxTranslator):
         category = self.escape_arg(smart_capwords(name))
         self.body.append('\n%s {%s} ' % (self.at_deffnx, category))
         self.at_deffnx = '@deffnx'
-        self.desc_type_name = name
+        self.desc_type_name: Optional[str] = name
 
     def depart_desc_signature(self, node: Element) -> None:
         self.body.append("\n")

tests/test_ext_autodoc.py

@@ -808,7 +808,7 @@ def test_autodoc_imported_members(app):
                "imported-members": None,
                "ignore-module-all": None}
     actual = do_autodoc(app, 'module', 'target', options)
-    assert '.. py:function:: save_traceback(app: Sphinx) -> str' in actual
+    assert '.. py:function:: save_traceback(app: ~typing.Optional[Sphinx]) -> str' in actual
 
 
 @pytest.mark.sphinx('html', testroot='ext-autodoc')