diff --git a/CHANGES b/CHANGES
index d7767ec6c..b19fb43de 100644
--- a/CHANGES
+++ b/CHANGES
@@ -99,9 +99,12 @@ Deprecated
----------
* pending_xref node for viewcode extension
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected``
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
* ``sphinx.builders.linkcheck.node_line_or_0()``
* ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()``
* ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter``
@@ -148,6 +151,7 @@ Features added
dedent via no-argument ``:dedent:`` option
* C++, also hyperlink operator overloads in expressions and alias declarations.
* #8247: Allow production lists to refer to tokens from other production groups
+* #8813: Show what extension (or module) caused it on errors on event handler
Bugs fixed
----------
diff --git a/doc/_themes/sphinx13/layout.html b/doc/_themes/sphinx13/layout.html
index 238fb52b7..b7f7c1424 100644
--- a/doc/_themes/sphinx13/layout.html
+++ b/doc/_themes/sphinx13/layout.html
@@ -67,7 +67,7 @@
Home
Get it
Docs
- Extend
+ Extend
diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst
index 117972a36..c58cb2038 100644
--- a/doc/extdev/deprecated.rst
+++ b/doc/extdev/deprecated.rst
@@ -72,6 +72,16 @@ The following is a list of deprecated interfaces.
- 5.0
- ``sphinx.ext.viewcode.viewcode_anchor``
+ * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
+ - 3.5
+ - 5.0
+ - N/A
+
+ * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
+ - 3.5
+ - 5.0
+ - N/A
+
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
- 3.5
- 5.0
@@ -87,6 +97,11 @@ The following is a list of deprecated interfaces.
- 5.0
- N/A
+ * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
+ - 3.5
+ - 5.0
+ - N/A
+
* - ``sphinx.builders.linkcheck.node_line_or_0()``
- 3.5
- 5.0
diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py
index 70cd302d5..900346def 100644
--- a/sphinx/builders/linkcheck.py
+++ b/sphinx/builders/linkcheck.py
@@ -12,14 +12,14 @@ import json
import queue
import re
import socket
-import threading
import time
import warnings
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from html.parser import HTMLParser
from os import path
-from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast
+from threading import Thread
+from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple, cast
from urllib.parse import unquote, urlparse
from docutils import nodes
@@ -44,6 +44,12 @@ Hyperlink = NamedTuple('Hyperlink', (('next_check', float),
('uri', Optional[str]),
('docname', Optional[str]),
('lineno', Optional[int])))
+CheckResult = NamedTuple('CheckResult', (('uri', str),
+ ('docname', str),
+ ('lineno', int),
+ ('status', str),
+ ('message', str),
+ ('code', int)))
RateLimit = NamedTuple('RateLimit', (('delay', float), ('next_check', float)))
DEFAULT_REQUEST_HEADERS = {
@@ -108,11 +114,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
def init(self) -> None:
self.hyperlinks = {} # type: Dict[str, Hyperlink]
- self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
- self.anchors_ignore = [re.compile(x)
- for x in self.config.linkcheck_anchors_ignore]
- self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
- in self.config.linkcheck_auth]
self._good = set() # type: Set[str]
self._broken = {} # type: Dict[str, str]
self._redirected = {} # type: Dict[str, Tuple[str, int]]
@@ -123,15 +124,43 @@ class CheckExternalLinksBuilder(DummyBuilder):
self.rate_limits = {} # type: Dict[str, RateLimit]
self.wqueue = queue.PriorityQueue() # type: queue.PriorityQueue
self.rqueue = queue.Queue() # type: queue.Queue
- self.workers = [] # type: List[threading.Thread]
+ self.workers = [] # type: List[Thread]
for i in range(self.config.linkcheck_workers):
- thread = threading.Thread(target=self.check_thread, daemon=True)
+ thread = HyperlinkAvailabilityCheckWorker(self)
thread.start()
self.workers.append(thread)
def is_ignored_uri(self, uri: str) -> bool:
return any(pat.match(uri) for pat in self.to_ignore)
+ @property
+ def anchors_ignore(self) -> List[Pattern]:
+ warnings.warn(
+ "%s.%s is deprecated." % (self.__class__.__name__, "anchors_ignore"),
+ RemovedInSphinx50Warning,
+ stacklevel=2,
+ )
+ return [re.compile(x) for x in self.config.linkcheck_anchors_ignore]
+
+ @property
+ def auth(self) -> List[Tuple[Pattern, Any]]:
+ warnings.warn(
+ "%s.%s is deprecated." % (self.__class__.__name__, "auth"),
+ RemovedInSphinx50Warning,
+ stacklevel=2,
+ )
+ return [(re.compile(pattern), auth_info) for pattern, auth_info
+ in self.config.linkcheck_auth]
+
+ @property
+ def to_ignore(self) -> List[Pattern]:
+ warnings.warn(
+ "%s.%s is deprecated." % (self.__class__.__name__, "to_ignore"),
+ RemovedInSphinx50Warning,
+ stacklevel=2,
+ )
+ return [re.compile(x) for x in self.config.linkcheck_ignore]
+
@property
def good(self) -> Set[str]:
warnings.warn(
@@ -160,6 +189,136 @@ class CheckExternalLinksBuilder(DummyBuilder):
return self._redirected
def check_thread(self) -> None:
+ warnings.warn(
+ "%s.%s is deprecated." % (self.__class__.__name__, "check_thread"),
+ RemovedInSphinx50Warning,
+ stacklevel=2,
+ )
+ # do nothing.
+
+ def limit_rate(self, response: Response) -> Optional[float]:
+ warnings.warn(
+ "%s.%s is deprecated." % (self.__class__.__name__, "limit_rate"),
+ RemovedInSphinx50Warning,
+ stacklevel=2,
+ )
+ return HyperlinkAvailabilityCheckWorker(self).limit_rate(response)
+
+ def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
+ uri, docname, lineno, status, info, code = result
+
+ filename = self.env.doc2path(docname, None)
+ linkstat = dict(filename=filename, lineno=lineno,
+ status=status, code=code, uri=uri,
+ info=info)
+ if status == 'unchecked':
+ self.write_linkstat(linkstat)
+ return
+ if status == 'working' and info == 'old':
+ self.write_linkstat(linkstat)
+ return
+ if lineno:
+ logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
+ if status == 'ignored':
+ if info:
+ logger.info(darkgray('-ignored- ') + uri + ': ' + info)
+ else:
+ logger.info(darkgray('-ignored- ') + uri)
+ self.write_linkstat(linkstat)
+ elif status == 'local':
+ logger.info(darkgray('-local- ') + uri)
+ self.write_entry('local', docname, filename, lineno, uri)
+ self.write_linkstat(linkstat)
+ elif status == 'working':
+ logger.info(darkgreen('ok ') + uri + info)
+ self.write_linkstat(linkstat)
+ elif status == 'broken':
+ if self.app.quiet or self.app.warningiserror:
+ logger.warning(__('broken link: %s (%s)'), uri, info,
+ location=(filename, lineno))
+ else:
+ logger.info(red('broken ') + uri + red(' - ' + info))
+ self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
+ self.write_linkstat(linkstat)
+ elif status == 'redirected':
+ try:
+ text, color = {
+ 301: ('permanently', purple),
+ 302: ('with Found', purple),
+ 303: ('with See Other', purple),
+ 307: ('temporarily', turquoise),
+ 308: ('permanently', purple),
+ }[code]
+ except KeyError:
+ text, color = ('with unknown code', purple)
+ linkstat['text'] = text
+ logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info))
+ self.write_entry('redirected ' + text, docname, filename,
+ lineno, uri + ' to ' + info)
+ self.write_linkstat(linkstat)
+ else:
+ raise ValueError("Unknown status %s." % status)
+
+ def write_entry(self, what: str, docname: str, filename: str, line: int,
+ uri: str) -> None:
+ self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
+
+ def write_linkstat(self, data: dict) -> None:
+ self.json_outfile.write(json.dumps(data))
+ self.json_outfile.write('\n')
+
+ def finish(self) -> None:
+ logger.info('')
+
+ with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
+ open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
+ total_links = 0
+ for hyperlink in self.hyperlinks.values():
+ if self.is_ignored_uri(hyperlink.uri):
+ self.process_result(
+ CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
+ 'ignored', '', 0))
+ else:
+ self.wqueue.put(hyperlink, False)
+ total_links += 1
+
+ done = 0
+ while done < total_links:
+ self.process_result(self.rqueue.get())
+ done += 1
+
+ if self._broken:
+ self.app.statuscode = 1
+
+ self.wqueue.join()
+ # Shutdown threads.
+ for worker in self.workers:
+ self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
+
+
+class HyperlinkAvailabilityCheckWorker(Thread):
+ """A worker class for checking the availability of hyperlinks."""
+
+ def __init__(self, builder: CheckExternalLinksBuilder) -> None:
+ self.config = builder.config
+ self.env = builder.env
+ self.rate_limits = builder.rate_limits
+ self.rqueue = builder.rqueue
+ self.wqueue = builder.wqueue
+
+ self.anchors_ignore = [re.compile(x)
+ for x in self.config.linkcheck_anchors_ignore]
+ self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
+ in self.config.linkcheck_auth]
+ self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
+
+ self._good = builder._good
+ self._broken = builder._broken
+ self._redirected = builder._redirected
+
+ super().__init__(daemon=True)
+
+ def run(self) -> None:
kwargs = {}
if self.config.linkcheck_timeout:
kwargs['timeout'] = self.config.linkcheck_timeout
@@ -372,96 +531,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
self.rate_limits[netloc] = RateLimit(delay, next_check)
return next_check
- def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
- uri, docname, lineno, status, info, code = result
-
- filename = self.env.doc2path(docname, None)
- linkstat = dict(filename=filename, lineno=lineno,
- status=status, code=code, uri=uri,
- info=info)
- if status == 'unchecked':
- self.write_linkstat(linkstat)
- return
- if status == 'working' and info == 'old':
- self.write_linkstat(linkstat)
- return
- if lineno:
- logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
- if status == 'ignored':
- if info:
- logger.info(darkgray('-ignored- ') + uri + ': ' + info)
- else:
- logger.info(darkgray('-ignored- ') + uri)
- self.write_linkstat(linkstat)
- elif status == 'local':
- logger.info(darkgray('-local- ') + uri)
- self.write_entry('local', docname, filename, lineno, uri)
- self.write_linkstat(linkstat)
- elif status == 'working':
- logger.info(darkgreen('ok ') + uri + info)
- self.write_linkstat(linkstat)
- elif status == 'broken':
- if self.app.quiet or self.app.warningiserror:
- logger.warning(__('broken link: %s (%s)'), uri, info,
- location=(filename, lineno))
- else:
- logger.info(red('broken ') + uri + red(' - ' + info))
- self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
- self.write_linkstat(linkstat)
- elif status == 'redirected':
- try:
- text, color = {
- 301: ('permanently', purple),
- 302: ('with Found', purple),
- 303: ('with See Other', purple),
- 307: ('temporarily', turquoise),
- 308: ('permanently', purple),
- }[code]
- except KeyError:
- text, color = ('with unknown code', purple)
- linkstat['text'] = text
- logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info))
- self.write_entry('redirected ' + text, docname, filename,
- lineno, uri + ' to ' + info)
- self.write_linkstat(linkstat)
- else:
- raise ValueError("Unknown status %s." % status)
-
- def write_entry(self, what: str, docname: str, filename: str, line: int,
- uri: str) -> None:
- self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
-
- def write_linkstat(self, data: dict) -> None:
- self.json_outfile.write(json.dumps(data))
- self.json_outfile.write('\n')
-
- def finish(self) -> None:
- logger.info('')
-
- with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
- open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
- total_links = 0
- for hyperlink in self.hyperlinks.values():
- if self.is_ignored_uri(hyperlink.uri):
- self.process_result((hyperlink.uri, hyperlink.docname, hyperlink.lineno,
- 'ignored', '', 0))
- else:
- self.wqueue.put(hyperlink, False)
- total_links += 1
-
- done = 0
- while done < total_links:
- self.process_result(self.rqueue.get())
- done += 1
-
- if self._broken:
- self.app.statuscode = 1
-
- self.wqueue.join()
- # Shutdown threads.
- for worker in self.workers:
- self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
-
class HyperlinkCollector(SphinxPostTransform):
builders = ('linkcheck',)
diff --git a/sphinx/errors.py b/sphinx/errors.py
index c632d8dbc..3e84b6b88 100644
--- a/sphinx/errors.py
+++ b/sphinx/errors.py
@@ -47,12 +47,19 @@ class ApplicationError(SphinxError):
class ExtensionError(SphinxError):
"""Extension error."""
- category = 'Extension error'
- def __init__(self, message: str, orig_exc: Exception = None) -> None:
+ def __init__(self, message: str, orig_exc: Exception = None, modname: str = None) -> None:
super().__init__(message)
self.message = message
self.orig_exc = orig_exc
+ self.modname = modname
+
+ @property
+ def category(self) -> str: # type: ignore
+ if self.modname:
+ return 'Extension error (%s)' % self.modname
+ else:
+ return 'Extension error'
def __repr__(self) -> str:
if self.orig_exc:
diff --git a/sphinx/events.py b/sphinx/events.py
index a29d551b7..881882b41 100644
--- a/sphinx/events.py
+++ b/sphinx/events.py
@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple, Tuple,
from sphinx.errors import ExtensionError, SphinxError
from sphinx.locale import __
from sphinx.util import logging
+from sphinx.util.inspect import safe_getattr
if TYPE_CHECKING:
from sphinx.application import Sphinx
@@ -104,8 +105,9 @@ class EventManager:
except SphinxError:
raise
except Exception as exc:
+ modname = safe_getattr(listener.handler, '__module__', None)
raise ExtensionError(__("Handler %r for event %r threw an exception") %
- (listener.handler, name), exc) from exc
+ (listener.handler, name), exc, modname=modname) from exc
return results
def emit_firstresult(self, name: str, *args: Any,
diff --git a/sphinx/testing/path.py b/sphinx/testing/path.py
index efc5e8f63..e8509d5ba 100644
--- a/sphinx/testing/path.py
+++ b/sphinx/testing/path.py
@@ -18,6 +18,17 @@ from sphinx.deprecation import RemovedInSphinx50Warning
FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
+def getumask() -> int:
+ """Get current umask value"""
+ umask = os.umask(0) # Note: Change umask value temporarily to obtain it
+ os.umask(umask)
+
+ return umask
+
+
+UMASK = getumask()
+
+
class path(str):
"""
Represents a path which behaves like a string.
@@ -98,6 +109,16 @@ class path(str):
pointed to by the symbolic links are copied.
"""
shutil.copytree(self, destination, symlinks=symlinks)
+ if os.environ.get('SPHINX_READONLY_TESTDIR'):
+ # If source tree is marked read-only (e.g. because it is on a read-only
+ # filesystem), `shutil.copytree` will mark the destination as read-only
+ # as well. To avoid failures when adding additional files/directories
+ # to the destination tree, ensure destination directories are not marked
+ # read-only.
+ for root, dirs, files in os.walk(destination):
+ os.chmod(root, 0o755 & ~UMASK)
+ for name in files:
+ os.chmod(os.path.join(root, name), 0o644 & ~UMASK)
def movetree(self, destination: str) -> None:
"""
diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty
index fa6695ae4..6b7e5dcce 100644
--- a/sphinx/texinputs/sphinx.sty
+++ b/sphinx/texinputs/sphinx.sty
@@ -173,7 +173,7 @@
% FIXME: this is unrelated to an option, move this elsewhere
% To allow hyphenation of first word in narrow contexts; no option,
% customization to be done via 'preamble' key
-\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip}
+\newcommand*\sphinxAtStartPar{\hskip\z@skip}
% No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex
\ifdefined\directlua\let\sphinxAtStartPar\@empty\fi
% user interface: options can be changed midway in a document!
diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py
index 60b62435c..e297d42c4 100644
--- a/tests/test_build_linkcheck.py
+++ b/tests/test_build_linkcheck.py
@@ -21,7 +21,8 @@ from unittest import mock
import pytest
import requests
-from sphinx.builders.linkcheck import CheckExternalLinksBuilder, RateLimit
+from sphinx.builders.linkcheck import (CheckExternalLinksBuilder,
+ HyperlinkAvailabilityCheckWorker, RateLimit)
from sphinx.util.console import strip_colors
from .utils import CERT_FILE, http_server, https_server
@@ -536,40 +537,50 @@ class FakeResponse:
def test_limit_rate_default_sleep(app):
checker = CheckExternalLinksBuilder(app)
+ checker.init()
checker.rate_limits = {}
+ worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0):
- next_check = checker.limit_rate(FakeResponse())
+ next_check = worker.limit_rate(FakeResponse())
assert next_check == 60.0
def test_limit_rate_user_max_delay(app):
app.config.linkcheck_rate_limit_timeout = 0.0
checker = CheckExternalLinksBuilder(app)
+ checker.init()
checker.rate_limits = {}
- next_check = checker.limit_rate(FakeResponse())
+ worker = HyperlinkAvailabilityCheckWorker(checker)
+ next_check = worker.limit_rate(FakeResponse())
assert next_check is None
def test_limit_rate_doubles_previous_wait_time(app):
checker = CheckExternalLinksBuilder(app)
+ checker.init()
checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
+ worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0):
- next_check = checker.limit_rate(FakeResponse())
+ next_check = worker.limit_rate(FakeResponse())
assert next_check == 120.0
def test_limit_rate_clips_wait_time_to_max_time(app):
checker = CheckExternalLinksBuilder(app)
+ checker.init()
app.config.linkcheck_rate_limit_timeout = 90.0
checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
+ worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0):
- next_check = checker.limit_rate(FakeResponse())
+ next_check = worker.limit_rate(FakeResponse())
assert next_check == 90.0
def test_limit_rate_bails_out_after_waiting_max_time(app):
checker = CheckExternalLinksBuilder(app)
+ checker.init()
app.config.linkcheck_rate_limit_timeout = 90.0
checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)}
- next_check = checker.limit_rate(FakeResponse())
+ worker = HyperlinkAvailabilityCheckWorker(checker)
+ next_check = worker.limit_rate(FakeResponse())
assert next_check is None