Add `sphinx._cli` (#10877)

This is the first step towards a new ``sphinx`` command.

.ruff.toml

@@ -386,6 +386,9 @@ select = [
 # from .flake8
 "sphinx/*" = ["E241"]
 
+# whitelist ``print`` for stdout messages
+"sphinx/_cli/__init__.py" = ["T201"]
+
 # whitelist ``print`` for stdout messages
 "sphinx/cmd/build.py" = ["T201"]
 "sphinx/cmd/make_mode.py" = ["T201"]
@@ -435,6 +438,7 @@ forced-separate = [
 preview = true
 quote-style = "single"
 exclude = [
+    "sphinx/_cli/*",
     "sphinx/addnodes.py",
     "sphinx/application.py",
     "sphinx/builders/*",

sphinx/_cli/__init__.py

@@ -0,0 +1,296 @@
+"""Base 'sphinx' command.
+
+Subcommands are loaded lazily from the ``_COMMANDS`` table for performance.
+
+All subcommand modules must define three attributes:
+
+- ``parser_description``, a description of the subcommand. The first paragraph
+  is taken as the short description for the command.
+- ``set_up_parser``, a callable taking and returning an ``ArgumentParser``. This
+  function is responsible for adding options and arguments to the subcommand's
+  parser.
+- ``run``, a callable taking parsed arguments and returning an exit code. This
+  function is responsible for running the main body of the subcommand and
+  returning the exit status.
+
+The entire ``sphinx._cli`` namespace is private, only the command line interface
+has backwards-compatability guarantees.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import locale
+import sys
+from typing import TYPE_CHECKING
+
+from sphinx._cli.util.colour import (
+    bold,
+    disable_colour,
+    enable_colour,
+    terminal_supports_colour,
+    underline,
+)
+from sphinx.locale import __, init_console
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator, Sequence
+    from typing import NoReturn
+
+    _PARSER_SETUP = Callable[[argparse.ArgumentParser], argparse.ArgumentParser]
+    _RUNNER = Callable[[argparse.Namespace], int]
+
+    from typing import Protocol
+
+    class _SubcommandModule(Protocol):
+        parser_description: str
+        set_up_parser: _PARSER_SETUP  # takes and returns argument parser
+        run: _RUNNER  # takes parsed args, returns exit code
+
+
+# Map of command name to import path.
+_COMMANDS: dict[str, str] = {
+}
+
+
+def _load_subcommand_descriptions() -> Iterator[tuple[str, str]]:
+    for command, module_name in _COMMANDS.items():
+        module: _SubcommandModule = importlib.import_module(module_name)
+        try:
+            description = module.parser_description
+        except AttributeError:
+            # log an error here, but don't fail the full enumeration
+            print(f"Failed to load the description for {command}", file=sys.stderr)
+        else:
+            yield command, description.split('\n\n', 1)[0]
+
+
+class _RootArgumentParser(argparse.ArgumentParser):
+    def format_help(self) -> str:
+        help_fragments: list[str] = [
+            bold(underline(__('Usage:'))),
+            ' ',
+            __('{0} [OPTIONS] <COMMAND> [<ARGS>]').format(bold(self.prog)),
+            '\n',
+            '\n',
+            __('  The Sphinx documentation generator.'),
+            '\n',
+        ]
+
+        if commands := list(_load_subcommand_descriptions()):
+            command_max_length = min(max(map(len, next(zip(*commands), ()))), 22)
+            help_fragments += [
+                '\n',
+                bold(underline(__('Commands:'))),
+                '\n',
+            ]
+            help_fragments += [
+                f'  {command_name: <{command_max_length}}  {command_desc}'
+                for command_name, command_desc in commands
+            ]
+            help_fragments.append('\n')
+
+        # self._action_groups[1] is self._optionals
+        # Uppercase the title of the Optionals group
+        self._optionals.title = __('Options')
+        for argument_group in self._action_groups[1:]:
+            if arguments := [action for action in argument_group._group_actions
+                             if action.help != argparse.SUPPRESS]:
+                help_fragments += self._format_optional_arguments(
+                    arguments,
+                    argument_group.title or '',
+                )
+
+        help_fragments += [
+            '\n',
+            __('For more information, visit https://www.sphinx-doc.org/en/master/man/.'),
+            '\n',
+        ]
+        return ''.join(help_fragments)
+
+    def _format_optional_arguments(
+        self,
+        actions: Iterable[argparse.Action],
+        title: str,
+    ) -> Iterator[str]:
+        yield '\n'
+        yield bold(underline(title + ':'))
+        yield '\n'
+
+        for action in actions:
+            prefix = '    ' * all(o[1] == '-' for o in action.option_strings)
+            opt = prefix + '  ' + ', '.join(map(bold, action.option_strings))
+            if action.nargs != 0:
+                opt += ' ' + self._format_metavar(
+                    action.nargs, action.metavar, action.choices, action.dest,
+                )
+            yield opt
+            yield '\n'
+            if action_help := (action.help or '').strip():
+                yield from (f'        {line}\n' for line in action_help.splitlines())
+
+    @staticmethod
+    def _format_metavar(
+        nargs: int | str | None,
+        metavar: str | tuple[str, ...] | None,
+        choices: Iterable[str] | None,
+        dest: str,
+    ) -> str:
+        if metavar is None:
+            if choices is not None:
+                metavar = '{' + ', '.join(sorted(choices)) + '}'
+            else:
+                metavar = dest.upper()
+        if nargs is None:
+            return f'{metavar}'
+        elif nargs == argparse.OPTIONAL:
+            return f'[{metavar}]'
+        elif nargs == argparse.ZERO_OR_MORE:
+            if len(metavar) == 2:
+                return f'[{metavar[0]} [{metavar[1]} ...]]'
+            else:
+                return f'[{metavar} ...]'
+        elif nargs == argparse.ONE_OR_MORE:
+            return f'{metavar} [{metavar} ...]'
+        elif nargs == argparse.REMAINDER:
+            return '...'
+        elif nargs == argparse.PARSER:
+            return f'{metavar} ...'
+        msg = 'invalid nargs value'
+        raise ValueError(msg)
+
+    def error(self, message: str) -> NoReturn:
+        sys.stderr.write(__(
+            '{0}: error: {1}\n'
+            "Run '{0} --help' for information"  # NoQA: COM812
+        ).format(self.prog, message))
+        raise SystemExit(2)
+
+
+def _create_parser() -> _RootArgumentParser:
+    parser = _RootArgumentParser(
+        prog='sphinx',
+        description=__('   Manage documentation with Sphinx.'),
+        epilog=__('For more information, visit https://www.sphinx-doc.org/en/master/man/.'),
+        add_help=False,
+        allow_abbrev=False,
+    )
+    parser.add_argument(
+        '-V', '--version',
+        action='store_true',
+        default=argparse.SUPPRESS,
+        help=__('Show the version and exit.'),
+    )
+    parser.add_argument(
+        '-h', '-?', '--help',
+        action='store_true',
+        default=argparse.SUPPRESS,
+        help=__('Show this message and exit.'),
+    )
+
+    # logging control
+    log_control = parser.add_argument_group(__('Logging'))
+    log_control.add_argument(
+        '-v', '--verbose',
+        action='count',
+        dest='verbosity',
+        default=0,
+        help=__('Increase verbosity (can be repeated)'),
+    )
+    log_control.add_argument(
+        '-q', '--quiet',
+        action='store_const',
+        dest='verbosity',
+        const=-1,
+        help=__('Only print errors and warnings.'),
+    )
+    log_control.add_argument(
+        '--silent',
+        action='store_const',
+        dest='verbosity',
+        const=-2,
+        help=__('No output at all'),
+    )
+
+    parser.add_argument(
+        'COMMAND',
+        nargs=argparse.REMAINDER,
+        metavar=__('<command>'),
+    )
+    return parser
+
+
+def _parse_command(argv: Sequence[str] = ()) -> tuple[str, Sequence[str]]:
+    parser = _create_parser()
+    args = parser.parse_args(argv)
+    command_name, *command_argv = args.COMMAND or ('help',)
+    command_name = command_name.lower()
+
+    if terminal_supports_colour():
+        enable_colour()
+    else:
+        disable_colour()
+
+    # Handle '--version' or '-V' passed to the main command or any subcommand
+    if 'version' in args or {'-V', '--version'}.intersection(command_argv):
+        from sphinx import __display_version__
+        sys.stderr.write(f'sphinx {__display_version__}\n')
+        raise SystemExit(0)
+
+    # Handle '--help' or '-h' passed to the main command (subcommands may have
+    # their own help text)
+    if 'help' in args or command_name == 'help':
+        sys.stderr.write(parser.format_help())
+        raise SystemExit(0)
+
+    if command_name not in _COMMANDS:
+        sys.stderr.write(__(f'sphinx: {command_name!r} is not a sphinx command. '
+                            "See 'sphinx --help'.\n"))
+        raise SystemExit(2)
+
+    return command_name, command_argv
+
+
+def _load_subcommand(command_name: str) -> tuple[str, _PARSER_SETUP, _RUNNER]:
+    try:
+        module: _SubcommandModule = importlib.import_module(_COMMANDS[command_name])
+    except KeyError:
+        msg = f'invalid command name {command_name!r}.'
+        raise ValueError(msg) from None
+    return module.parser_description, module.set_up_parser, module.run
+
+
+def _create_sub_parser(
+    command_name: str,
+    description: str,
+    parser_setup: _PARSER_SETUP,
+) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=f'sphinx {command_name}',
+        description=description,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        allow_abbrev=False,
+    )
+    return parser_setup(parser)
+
+
+def run(argv: Sequence[str] = (), /) -> int:
+    locale.setlocale(locale.LC_ALL, '')
+    init_console()
+
+    argv = argv or sys.argv[1:]
+    try:
+        cmd_name, cmd_argv = _parse_command(argv)
+        cmd_description, set_up_parser, runner = _load_subcommand(cmd_name)
+        cmd_parser = _create_sub_parser(cmd_name, cmd_description, set_up_parser)
+        cmd_args = cmd_parser.parse_args(cmd_argv)
+        return runner(cmd_args)
+    except SystemExit as exc:
+        return exc.code  # type: ignore[return-value]
+    except (Exception, KeyboardInterrupt):
+        return 2
+
+
+if __name__ == '__main__':
+    raise SystemExit(run())

sphinx/_cli/util/colour.py

@@ -0,0 +1,103 @@
+"""Format coloured console output."""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections.abc import Callable  # NoQA: TCH003
+
+if sys.platform == 'win32':
+    import colorama
+
+
+_COLOURING_DISABLED = True
+
+
+def terminal_supports_colour() -> bool:
+    """Return True if coloured terminal output is supported."""
+    if 'NO_COLOUR' in os.environ or 'NO_COLOR' in os.environ:
+        return False
+    if 'FORCE_COLOUR' in os.environ or 'FORCE_COLOR' in os.environ:
+        return True
+
+    try:
+        if not sys.stdout.isatty():
+            return False
+    except (AttributeError, ValueError):
+        # Handle cases where .isatty() is not defined, or where e.g.
+        # "ValueError: I/O operation on closed file" is raised
+        return False
+
+    # Do not colour output if on a dumb terminal
+    return os.environ.get('TERM', 'unknown').lower() not in {'dumb', 'unknown'}
+
+
+def disable_colour() -> None:
+    global _COLOURING_DISABLED
+    _COLOURING_DISABLED = True
+    if sys.platform == 'win32':
+        colorama.deinit()
+
+
+def enable_colour() -> None:
+    global _COLOURING_DISABLED
+    _COLOURING_DISABLED = False
+    if sys.platform == 'win32':
+        colorama.init()
+
+
+def colourise(colour_name: str, text: str, /) -> str:
+    if _COLOURING_DISABLED:
+        return text
+    return globals()[colour_name](text)
+
+
+def _create_colour_func(escape_code: str, /) -> Callable[[str], str]:
+    def inner(text: str) -> str:
+        if _COLOURING_DISABLED:
+            return text
+        return f'\x1b[{escape_code}m{text}\x1b[39;49;00m'
+    return inner
+
+
+# Wrap escape sequence with ``\1`` and ``\2`` to let readline know
+# that the colour escape codes are non-printable characters
+# [ https://tiswww.case.edu/php/chet/readline/readline.html ]
+#
+# Note: This does not work well in Windows
+# (see https://github.com/sphinx-doc/sphinx/pull/5059)
+if sys.platform == 'win32':
+    _create_input_mode_colour_func = _create_colour_func
+else:
+    def _create_input_mode_colour_func(escape_code: str, /) -> Callable[[str], str]:
+        def inner(text: str) -> str:
+            if _COLOURING_DISABLED:
+                return text
+            return f'\x01\x1b[{escape_code}m\x02{text}\x01\x1b[39;49;00m\x02'
+        return inner
+
+
+reset = _create_colour_func('39;49;00')
+bold = _create_colour_func('01')
+faint = _create_colour_func('02')
+standout = _create_colour_func('03')
+underline = _create_colour_func('04')
+blink = _create_colour_func('05')
+
+black = _create_colour_func('30')
+darkred = _create_colour_func('31')
+darkgreen = _create_colour_func('32')
+brown = _create_colour_func('33')
+darkblue = _create_colour_func('34')
+purple = _create_colour_func('35')
+turquoise = _create_colour_func('36')
+lightgray = _create_colour_func('37')
+
+darkgray = _create_colour_func('90')
+red = _create_colour_func('91')
+green = _create_colour_func('92')
+yellow = _create_colour_func('93')
+blue = _create_colour_func('94')
+fuchsia = _create_colour_func('95')
+teal = _create_colour_func('96')
+white = _create_colour_func('97')

sphinx/_cli/util/errors.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import re
+import sys
+import tempfile
+from typing import TYPE_CHECKING, TextIO
+
+from sphinx.errors import SphinxParallelError
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+_ANSI_COLOUR_CODES: re.Pattern[str] = re.compile('\x1b.*?m')
+
+
+def terminal_safe(s: str, /) -> str:
+    """Safely encode a string for printing to the terminal."""
+    return s.encode('ascii', 'backslashreplace').decode('ascii')
+
+
+def strip_colors(s: str, /) -> str:
+    return _ANSI_COLOUR_CODES.sub('', s).strip()
+
+
+def error_info(messages: str, extensions: str, traceback: str) -> str:
+    import platform
+
+    import docutils
+    import jinja2
+    import pygments
+
+    import sphinx
+
+    return f"""\
+Versions
+========
+
+* Platform:         {sys.platform}; ({platform.platform()})
+* Python version:   {platform.python_version()} ({platform.python_implementation()})
+* Sphinx version:   {sphinx.__display_version__}
+* Docutils version: {docutils.__version__}
+* Jinja2 version:   {jinja2.__version__}
+* Pygments version: {pygments.__version__}
+
+Last Messages
+=============
+
+{messages}
+
+Loaded Extensions
+=================
+
+{extensions}
+
+Traceback
+=========
+
+{traceback}
+"""
+
+
+def save_traceback(app: Sphinx | None, exc: BaseException) -> str:
+    """Save the given exception's traceback in a temporary file."""
+    if isinstance(exc, SphinxParallelError):
+        exc_format = '(Error in parallel process)\n' + exc.traceback
+    else:
+        import traceback
+
+        exc_format = traceback.format_exc()
+
+    last_msgs = exts_list = ''
+    if app is not None:
+        extensions = app.extensions.values()
+        last_msgs = '\n'.join(f'* {strip_colors(s)}' for s in app.messagelog)
+        exts_list = '\n'.join(f'* {ext.name} ({ext.version})' for ext in extensions
+                              if ext.version != 'builtin')
+
+    with tempfile.NamedTemporaryFile(suffix='.log', prefix='sphinx-err-', delete=False) as f:
+        f.write(error_info(last_msgs, exts_list, exc_format).encode('utf-8'))
+
+    return f.name
+
+
+def handle_exception(
+    exception: BaseException,
+    /,
+    *,
+    stderr: TextIO = sys.stderr,
+    use_pdb: bool = False,
+    print_traceback: bool = False,
+    app: Sphinx | None = None,
+) -> None:
+    from bdb import BdbQuit
+    from traceback import TracebackException, print_exc
+
+    from docutils.utils import SystemMessage
+
+    from sphinx._cli.util.colour import red
+    from sphinx.errors import SphinxError
+    from sphinx.locale import __
+
+    if isinstance(exception, BdbQuit):
+        return
+
+    def print_err(*values: str) -> None:
+        print(*values, file=stderr)
+
+    def print_red(*values: str) -> None:
+        print_err(*map(red, values))
+
+    print_err()
+    if print_traceback or use_pdb:
+        print_exc(file=stderr)
+        print_err()
+
+    if use_pdb:
+        from pdb import post_mortem
+
+        print_red(__('Exception occurred, starting debugger:'))
+        post_mortem()
+        return
+
+    if isinstance(exception, KeyboardInterrupt):
+        print_err(__('Interrupted!'))
+        return
+
+    if isinstance(exception, SystemMessage):
+        print_red(__('reStructuredText markup error:'))
+        print_err(str(exception))
+        return
+
+    if isinstance(exception, SphinxError):
+        print_red(f'{exception.category}:')
+        print_err(str(exception))
+        return
+
+    if isinstance(exception, UnicodeError):
+        print_red(__('Encoding error:'))
+        print_err(str(exception))
+        return
+
+    if isinstance(exception, RecursionError):
+        print_red(__('Recursion error:'))
+        print_err(str(exception))
+        print_err()
+        print_err(__('This can happen with very large or deeply nested source '
+                     'files. You can carefully increase the default Python '
+                     'recursion limit of 1000 in conf.py with e.g.:'))
+        print_err('\n    import sys\n    sys.setrecursionlimit(1_500)\n')
+        return
+
+    # format an exception with traceback, but only the last frame.
+    te = TracebackException.from_exception(exception, limit=-1)
+    formatted_tb = te.stack.format()[-1] + ''.join(te.format_exception_only()).rstrip()
+
+    print_red(__('Exception occurred:'))
+    print_err(formatted_tb)
+    traceback_info_path = save_traceback(app, exception)
+    print_err(__('The full traceback has been saved in:'))
+    print_err(traceback_info_path)
+    print_err()
+    print_err(__('To report this error to the developers, please open an issue '
+                 'at <https://github.com/sphinx-doc/sphinx/issues/>. Thanks!'))
+    print_err(__('Please also report this if it was a user error, so '
+                 'that a better error message can be provided next time.'))