Merge branch '3.x'

CHANGES

@@ -99,9 +99,12 @@ Deprecated
 ----------
 
 * pending_xref node for viewcode extension
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
 * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
 * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good``
 * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected``
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
 * ``sphinx.builders.linkcheck.node_line_or_0()``
 * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()``
 * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter``
@@ -148,6 +151,7 @@ Features added
   dedent via no-argument ``:dedent:`` option
 * C++, also hyperlink operator overloads in expressions and alias declarations.
 * #8247: Allow production lists to refer to tokens from other production groups
+* #8813: Show what extension (or module) caused it on errors on event handler
 
 Bugs fixed
 ----------

doc/_themes/sphinx13/layout.html

@@ -67,7 +67,7 @@
     <li><a href="{{ pathto('index') }}">Home</a></li>
     <li><a href="{{ pathto('usage/installation') }}">Get it</a></li>
     <li><a href="{{ pathto('contents') }}">Docs</a></li>
-    <li><a href="{{ pathto('development') }}">Extend</a></li>
+    <li><a href="{{ pathto('development/index') }}">Extend</a></li>
   </ul>
   <div>
     <a href="{{ pathto('index') }}">

doc/extdev/deprecated.rst

@@ -72,6 +72,16 @@ The following is a list of deprecated interfaces.
      - 5.0
      - ``sphinx.ext.viewcode.viewcode_anchor``
 
+   * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
+     - 3.5
+     - 5.0
+     - N/A
+
+   * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
+     - 3.5
+     - 5.0
+     - N/A
+
    * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
      - 3.5
      - 5.0
@@ -87,6 +97,11 @@ The following is a list of deprecated interfaces.
      - 5.0
      - N/A
 
+   * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
+     - 3.5
+     - 5.0
+     - N/A
+
    * - ``sphinx.builders.linkcheck.node_line_or_0()``
      - 3.5
      - 5.0

sphinx/builders/linkcheck.py

@@ -12,14 +12,14 @@ import json
 import queue
 import re
 import socket
-import threading
 import time
 import warnings
 from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 from html.parser import HTMLParser
 from os import path
-from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast
+from threading import Thread
+from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple, cast
 from urllib.parse import unquote, urlparse
 
 from docutils import nodes
@@ -44,6 +44,12 @@ Hyperlink = NamedTuple('Hyperlink', (('next_check', float),
                                      ('uri', Optional[str]),
                                      ('docname', Optional[str]),
                                      ('lineno', Optional[int])))
+CheckResult = NamedTuple('CheckResult', (('uri', str),
+                                         ('docname', str),
+                                         ('lineno', int),
+                                         ('status', str),
+                                         ('message', str),
+                                         ('code', int)))
 RateLimit = NamedTuple('RateLimit', (('delay', float), ('next_check', float)))
 
 DEFAULT_REQUEST_HEADERS = {
@@ -108,11 +114,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
 
     def init(self) -> None:
         self.hyperlinks = {}    # type: Dict[str, Hyperlink]
-        self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
-        self.anchors_ignore = [re.compile(x)
-                               for x in self.config.linkcheck_anchors_ignore]
-        self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
-                     in self.config.linkcheck_auth]
         self._good = set()       # type: Set[str]
         self._broken = {}        # type: Dict[str, str]
         self._redirected = {}    # type: Dict[str, Tuple[str, int]]
@@ -123,15 +124,43 @@ class CheckExternalLinksBuilder(DummyBuilder):
         self.rate_limits = {}  # type: Dict[str, RateLimit]
         self.wqueue = queue.PriorityQueue()  # type: queue.PriorityQueue
         self.rqueue = queue.Queue()  # type: queue.Queue
-        self.workers = []  # type: List[threading.Thread]
+        self.workers = []  # type: List[Thread]
         for i in range(self.config.linkcheck_workers):
-            thread = threading.Thread(target=self.check_thread, daemon=True)
+            thread = HyperlinkAvailabilityCheckWorker(self)
             thread.start()
             self.workers.append(thread)
 
     def is_ignored_uri(self, uri: str) -> bool:
         return any(pat.match(uri) for pat in self.to_ignore)
 
+    @property
+    def anchors_ignore(self) -> List[Pattern]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "anchors_ignore"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return [re.compile(x) for x in self.config.linkcheck_anchors_ignore]
+
+    @property
+    def auth(self) -> List[Tuple[Pattern, Any]]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "auth"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return [(re.compile(pattern), auth_info) for pattern, auth_info
+                in self.config.linkcheck_auth]
+
+    @property
+    def to_ignore(self) -> List[Pattern]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "to_ignore"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return [re.compile(x) for x in self.config.linkcheck_ignore]
+
     @property
     def good(self) -> Set[str]:
         warnings.warn(
@@ -160,6 +189,136 @@ class CheckExternalLinksBuilder(DummyBuilder):
         return self._redirected
 
     def check_thread(self) -> None:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "check_thread"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        # do nothing.
+
+    def limit_rate(self, response: Response) -> Optional[float]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "limit_rate"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return HyperlinkAvailabilityCheckWorker(self).limit_rate(response)
+
+    def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
+        uri, docname, lineno, status, info, code = result
+
+        filename = self.env.doc2path(docname, None)
+        linkstat = dict(filename=filename, lineno=lineno,
+                        status=status, code=code, uri=uri,
+                        info=info)
+        if status == 'unchecked':
+            self.write_linkstat(linkstat)
+            return
+        if status == 'working' and info == 'old':
+            self.write_linkstat(linkstat)
+            return
+        if lineno:
+            logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
+        if status == 'ignored':
+            if info:
+                logger.info(darkgray('-ignored- ') + uri + ': ' + info)
+            else:
+                logger.info(darkgray('-ignored- ') + uri)
+            self.write_linkstat(linkstat)
+        elif status == 'local':
+            logger.info(darkgray('-local-   ') + uri)
+            self.write_entry('local', docname, filename, lineno, uri)
+            self.write_linkstat(linkstat)
+        elif status == 'working':
+            logger.info(darkgreen('ok        ') + uri + info)
+            self.write_linkstat(linkstat)
+        elif status == 'broken':
+            if self.app.quiet or self.app.warningiserror:
+                logger.warning(__('broken link: %s (%s)'), uri, info,
+                               location=(filename, lineno))
+            else:
+                logger.info(red('broken    ') + uri + red(' - ' + info))
+            self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
+            self.write_linkstat(linkstat)
+        elif status == 'redirected':
+            try:
+                text, color = {
+                    301: ('permanently', purple),
+                    302: ('with Found', purple),
+                    303: ('with See Other', purple),
+                    307: ('temporarily', turquoise),
+                    308: ('permanently', purple),
+                }[code]
+            except KeyError:
+                text, color = ('with unknown code', purple)
+            linkstat['text'] = text
+            logger.info(color('redirect  ') + uri + color(' - ' + text + ' to ' + info))
+            self.write_entry('redirected ' + text, docname, filename,
+                             lineno, uri + ' to ' + info)
+            self.write_linkstat(linkstat)
+        else:
+            raise ValueError("Unknown status %s." % status)
+
+    def write_entry(self, what: str, docname: str, filename: str, line: int,
+                    uri: str) -> None:
+        self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
+
+    def write_linkstat(self, data: dict) -> None:
+        self.json_outfile.write(json.dumps(data))
+        self.json_outfile.write('\n')
+
+    def finish(self) -> None:
+        logger.info('')
+
+        with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
+             open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
+            total_links = 0
+            for hyperlink in self.hyperlinks.values():
+                if self.is_ignored_uri(hyperlink.uri):
+                    self.process_result(
+                        CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
+                                    'ignored', '', 0))
+                else:
+                    self.wqueue.put(hyperlink, False)
+                    total_links += 1
+
+            done = 0
+            while done < total_links:
+                self.process_result(self.rqueue.get())
+                done += 1
+
+        if self._broken:
+            self.app.statuscode = 1
+
+        self.wqueue.join()
+        # Shutdown threads.
+        for worker in self.workers:
+            self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
+
+
+class HyperlinkAvailabilityCheckWorker(Thread):
+    """A worker class for checking the availability of hyperlinks."""
+
+    def __init__(self, builder: CheckExternalLinksBuilder) -> None:
+        self.config = builder.config
+        self.env = builder.env
+        self.rate_limits = builder.rate_limits
+        self.rqueue = builder.rqueue
+        self.wqueue = builder.wqueue
+
+        self.anchors_ignore = [re.compile(x)
+                               for x in self.config.linkcheck_anchors_ignore]
+        self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
+                     in self.config.linkcheck_auth]
+        self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
+
+        self._good = builder._good
+        self._broken = builder._broken
+        self._redirected = builder._redirected
+
+        super().__init__(daemon=True)
+
+    def run(self) -> None:
         kwargs = {}
         if self.config.linkcheck_timeout:
             kwargs['timeout'] = self.config.linkcheck_timeout
@@ -372,96 +531,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
         self.rate_limits[netloc] = RateLimit(delay, next_check)
         return next_check
 
-    def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
-        uri, docname, lineno, status, info, code = result
-
-        filename = self.env.doc2path(docname, None)
-        linkstat = dict(filename=filename, lineno=lineno,
-                        status=status, code=code, uri=uri,
-                        info=info)
-        if status == 'unchecked':
-            self.write_linkstat(linkstat)
-            return
-        if status == 'working' and info == 'old':
-            self.write_linkstat(linkstat)
-            return
-        if lineno:
-            logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
-        if status == 'ignored':
-            if info:
-                logger.info(darkgray('-ignored- ') + uri + ': ' + info)
-            else:
-                logger.info(darkgray('-ignored- ') + uri)
-            self.write_linkstat(linkstat)
-        elif status == 'local':
-            logger.info(darkgray('-local-   ') + uri)
-            self.write_entry('local', docname, filename, lineno, uri)
-            self.write_linkstat(linkstat)
-        elif status == 'working':
-            logger.info(darkgreen('ok        ') + uri + info)
-            self.write_linkstat(linkstat)
-        elif status == 'broken':
-            if self.app.quiet or self.app.warningiserror:
-                logger.warning(__('broken link: %s (%s)'), uri, info,
-                               location=(filename, lineno))
-            else:
-                logger.info(red('broken    ') + uri + red(' - ' + info))
-            self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
-            self.write_linkstat(linkstat)
-        elif status == 'redirected':
-            try:
-                text, color = {
-                    301: ('permanently', purple),
-                    302: ('with Found', purple),
-                    303: ('with See Other', purple),
-                    307: ('temporarily', turquoise),
-                    308: ('permanently', purple),
-                }[code]
-            except KeyError:
-                text, color = ('with unknown code', purple)
-            linkstat['text'] = text
-            logger.info(color('redirect  ') + uri + color(' - ' + text + ' to ' + info))
-            self.write_entry('redirected ' + text, docname, filename,
-                             lineno, uri + ' to ' + info)
-            self.write_linkstat(linkstat)
-        else:
-            raise ValueError("Unknown status %s." % status)
-
-    def write_entry(self, what: str, docname: str, filename: str, line: int,
-                    uri: str) -> None:
-        self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
-
-    def write_linkstat(self, data: dict) -> None:
-        self.json_outfile.write(json.dumps(data))
-        self.json_outfile.write('\n')
-
-    def finish(self) -> None:
-        logger.info('')
-
-        with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
-             open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
-            total_links = 0
-            for hyperlink in self.hyperlinks.values():
-                if self.is_ignored_uri(hyperlink.uri):
-                    self.process_result((hyperlink.uri, hyperlink.docname, hyperlink.lineno,
-                                         'ignored', '', 0))
-                else:
-                    self.wqueue.put(hyperlink, False)
-                    total_links += 1
-
-            done = 0
-            while done < total_links:
-                self.process_result(self.rqueue.get())
-                done += 1
-
-        if self._broken:
-            self.app.statuscode = 1
-
-        self.wqueue.join()
-        # Shutdown threads.
-        for worker in self.workers:
-            self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
-
 
 class HyperlinkCollector(SphinxPostTransform):
     builders = ('linkcheck',)

sphinx/errors.py

@@ -47,12 +47,19 @@ class ApplicationError(SphinxError):
 
 class ExtensionError(SphinxError):
     """Extension error."""
-    category = 'Extension error'
 
-    def __init__(self, message: str, orig_exc: Exception = None) -> None:
+    def __init__(self, message: str, orig_exc: Exception = None, modname: str = None) -> None:
         super().__init__(message)
         self.message = message
         self.orig_exc = orig_exc
+        self.modname = modname
+
+    @property
+    def category(self) -> str:  # type: ignore
+        if self.modname:
+            return 'Extension error (%s)' % self.modname
+        else:
+            return 'Extension error'
 
     def __repr__(self) -> str:
         if self.orig_exc:

sphinx/events.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple, Tuple,
 from sphinx.errors import ExtensionError, SphinxError
 from sphinx.locale import __
 from sphinx.util import logging
+from sphinx.util.inspect import safe_getattr
 
 if TYPE_CHECKING:
     from sphinx.application import Sphinx
@@ -104,8 +105,9 @@ class EventManager:
             except SphinxError:
                 raise
             except Exception as exc:
+                modname = safe_getattr(listener.handler, '__module__', None)
                 raise ExtensionError(__("Handler %r for event %r threw an exception") %
-                                     (listener.handler, name), exc) from exc
+                                     (listener.handler, name), exc, modname=modname) from exc
         return results
 
     def emit_firstresult(self, name: str, *args: Any,

sphinx/testing/path.py

@@ -18,6 +18,17 @@ from sphinx.deprecation import RemovedInSphinx50Warning
 FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
 
 
+def getumask() -> int:
+    """Get current umask value"""
+    umask = os.umask(0)  # Note: Change umask value temporarily to obtain it
+    os.umask(umask)
+
+    return umask
+
+
+UMASK = getumask()
+
+
 class path(str):
     """
     Represents a path which behaves like a string.
@@ -98,6 +109,16 @@ class path(str):
             pointed to by the symbolic links are copied.
         """
         shutil.copytree(self, destination, symlinks=symlinks)
+        if os.environ.get('SPHINX_READONLY_TESTDIR'):
+            # If source tree is marked read-only (e.g. because it is on a read-only
+            # filesystem), `shutil.copytree` will mark the destination as read-only
+            # as well.  To avoid failures when adding additional files/directories
+            # to the destination tree, ensure destination directories are not marked
+            # read-only.
+            for root, dirs, files in os.walk(destination):
+                os.chmod(root, 0o755 & ~UMASK)
+                for name in files:
+                    os.chmod(os.path.join(root, name), 0o644 & ~UMASK)
 
     def movetree(self, destination: str) -> None:
         """

sphinx/texinputs/sphinx.sty

@@ -173,7 +173,7 @@
 % FIXME: this is unrelated to an option, move this elsewhere
 % To allow hyphenation of first word in narrow contexts; no option,
 % customization to be done via 'preamble' key
-\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip}
+\newcommand*\sphinxAtStartPar{\hskip\z@skip}
 % No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex
 \ifdefined\directlua\let\sphinxAtStartPar\@empty\fi
 % user interface: options can be changed midway in a document!

tests/test_build_linkcheck.py

@@ -21,7 +21,8 @@ from unittest import mock
 import pytest
 import requests
 
-from sphinx.builders.linkcheck import CheckExternalLinksBuilder, RateLimit
+from sphinx.builders.linkcheck import (CheckExternalLinksBuilder,
+                                       HyperlinkAvailabilityCheckWorker, RateLimit)
 from sphinx.util.console import strip_colors
 
 from .utils import CERT_FILE, http_server, https_server
@@ -536,40 +537,50 @@ class FakeResponse:
 
 def test_limit_rate_default_sleep(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     checker.rate_limits = {}
+    worker = HyperlinkAvailabilityCheckWorker(checker)
     with mock.patch('time.time', return_value=0.0):
-        next_check = checker.limit_rate(FakeResponse())
+        next_check = worker.limit_rate(FakeResponse())
     assert next_check == 60.0
 
 
 def test_limit_rate_user_max_delay(app):
     app.config.linkcheck_rate_limit_timeout = 0.0
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     checker.rate_limits = {}
-    next_check = checker.limit_rate(FakeResponse())
+    worker = HyperlinkAvailabilityCheckWorker(checker)
+    next_check = worker.limit_rate(FakeResponse())
     assert next_check is None
 
 
 def test_limit_rate_doubles_previous_wait_time(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
+    worker = HyperlinkAvailabilityCheckWorker(checker)
     with mock.patch('time.time', return_value=0.0):
-        next_check = checker.limit_rate(FakeResponse())
+        next_check = worker.limit_rate(FakeResponse())
     assert next_check == 120.0
 
 
 def test_limit_rate_clips_wait_time_to_max_time(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     app.config.linkcheck_rate_limit_timeout = 90.0
     checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
+    worker = HyperlinkAvailabilityCheckWorker(checker)
     with mock.patch('time.time', return_value=0.0):
-        next_check = checker.limit_rate(FakeResponse())
+        next_check = worker.limit_rate(FakeResponse())
     assert next_check == 90.0
 
 
 def test_limit_rate_bails_out_after_waiting_max_time(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     app.config.linkcheck_rate_limit_timeout = 90.0
     checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)}
-    next_check = checker.limit_rate(FakeResponse())
+    worker = HyperlinkAvailabilityCheckWorker(checker)
+    next_check = worker.limit_rate(FakeResponse())
     assert next_check is None