Merge branch '3.x'

This commit is contained in:
Takeshi KOMIYA 2021-02-07 17:01:09 +09:00
commit 84458da828
9 changed files with 239 additions and 110 deletions

View File

@ -99,9 +99,12 @@ Deprecated
---------- ----------
* pending_xref node for viewcode extension * pending_xref node for viewcode extension
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
* ``sphinx.builders.linkcheck.node_line_or_0()`` * ``sphinx.builders.linkcheck.node_line_or_0()``
* ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()`` * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()``
* ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter`` * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter``
@ -148,6 +151,7 @@ Features added
dedent via no-argument ``:dedent:`` option dedent via no-argument ``:dedent:`` option
* C++, also hyperlink operator overloads in expressions and alias declarations. * C++, also hyperlink operator overloads in expressions and alias declarations.
* #8247: Allow production lists to refer to tokens from other production groups * #8247: Allow production lists to refer to tokens from other production groups
* #8813: Show what extension (or module) caused it on errors on event handler
Bugs fixed Bugs fixed
---------- ----------

View File

@ -67,7 +67,7 @@
<li><a href="{{ pathto('index') }}">Home</a></li> <li><a href="{{ pathto('index') }}">Home</a></li>
<li><a href="{{ pathto('usage/installation') }}">Get it</a></li> <li><a href="{{ pathto('usage/installation') }}">Get it</a></li>
<li><a href="{{ pathto('contents') }}">Docs</a></li> <li><a href="{{ pathto('contents') }}">Docs</a></li>
<li><a href="{{ pathto('development') }}">Extend</a></li> <li><a href="{{ pathto('development/index') }}">Extend</a></li>
</ul> </ul>
<div> <div>
<a href="{{ pathto('index') }}"> <a href="{{ pathto('index') }}">

View File

@ -72,6 +72,16 @@ The following is a list of deprecated interfaces.
- 5.0 - 5.0
- ``sphinx.ext.viewcode.viewcode_anchor`` - ``sphinx.ext.viewcode.viewcode_anchor``
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
- 3.5
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
- 3.5
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
- 3.5 - 3.5
- 5.0 - 5.0
@ -87,6 +97,11 @@ The following is a list of deprecated interfaces.
- 5.0 - 5.0
- N/A - N/A
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
- 3.5
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.node_line_or_0()`` * - ``sphinx.builders.linkcheck.node_line_or_0()``
- 3.5 - 3.5
- 5.0 - 5.0

View File

@ -12,14 +12,14 @@ import json
import queue import queue
import re import re
import socket import socket
import threading
import time import time
import warnings import warnings
from datetime import datetime, timezone from datetime import datetime, timezone
from email.utils import parsedate_to_datetime from email.utils import parsedate_to_datetime
from html.parser import HTMLParser from html.parser import HTMLParser
from os import path from os import path
from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast from threading import Thread
from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple, cast
from urllib.parse import unquote, urlparse from urllib.parse import unquote, urlparse
from docutils import nodes from docutils import nodes
@ -44,6 +44,12 @@ Hyperlink = NamedTuple('Hyperlink', (('next_check', float),
('uri', Optional[str]), ('uri', Optional[str]),
('docname', Optional[str]), ('docname', Optional[str]),
('lineno', Optional[int]))) ('lineno', Optional[int])))
CheckResult = NamedTuple('CheckResult', (('uri', str),
('docname', str),
('lineno', int),
('status', str),
('message', str),
('code', int)))
RateLimit = NamedTuple('RateLimit', (('delay', float), ('next_check', float))) RateLimit = NamedTuple('RateLimit', (('delay', float), ('next_check', float)))
DEFAULT_REQUEST_HEADERS = { DEFAULT_REQUEST_HEADERS = {
@ -108,11 +114,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
def init(self) -> None: def init(self) -> None:
self.hyperlinks = {} # type: Dict[str, Hyperlink] self.hyperlinks = {} # type: Dict[str, Hyperlink]
self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
self.anchors_ignore = [re.compile(x)
for x in self.config.linkcheck_anchors_ignore]
self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
in self.config.linkcheck_auth]
self._good = set() # type: Set[str] self._good = set() # type: Set[str]
self._broken = {} # type: Dict[str, str] self._broken = {} # type: Dict[str, str]
self._redirected = {} # type: Dict[str, Tuple[str, int]] self._redirected = {} # type: Dict[str, Tuple[str, int]]
@ -123,15 +124,43 @@ class CheckExternalLinksBuilder(DummyBuilder):
self.rate_limits = {} # type: Dict[str, RateLimit] self.rate_limits = {} # type: Dict[str, RateLimit]
self.wqueue = queue.PriorityQueue() # type: queue.PriorityQueue self.wqueue = queue.PriorityQueue() # type: queue.PriorityQueue
self.rqueue = queue.Queue() # type: queue.Queue self.rqueue = queue.Queue() # type: queue.Queue
self.workers = [] # type: List[threading.Thread] self.workers = [] # type: List[Thread]
for i in range(self.config.linkcheck_workers): for i in range(self.config.linkcheck_workers):
thread = threading.Thread(target=self.check_thread, daemon=True) thread = HyperlinkAvailabilityCheckWorker(self)
thread.start() thread.start()
self.workers.append(thread) self.workers.append(thread)
def is_ignored_uri(self, uri: str) -> bool: def is_ignored_uri(self, uri: str) -> bool:
return any(pat.match(uri) for pat in self.to_ignore) return any(pat.match(uri) for pat in self.to_ignore)
@property
def anchors_ignore(self) -> List[Pattern]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "anchors_ignore"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return [re.compile(x) for x in self.config.linkcheck_anchors_ignore]
@property
def auth(self) -> List[Tuple[Pattern, Any]]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "auth"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return [(re.compile(pattern), auth_info) for pattern, auth_info
in self.config.linkcheck_auth]
@property
def to_ignore(self) -> List[Pattern]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "to_ignore"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return [re.compile(x) for x in self.config.linkcheck_ignore]
@property @property
def good(self) -> Set[str]: def good(self) -> Set[str]:
warnings.warn( warnings.warn(
@ -160,6 +189,136 @@ class CheckExternalLinksBuilder(DummyBuilder):
return self._redirected return self._redirected
def check_thread(self) -> None: def check_thread(self) -> None:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "check_thread"),
RemovedInSphinx50Warning,
stacklevel=2,
)
# do nothing.
def limit_rate(self, response: Response) -> Optional[float]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "limit_rate"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return HyperlinkAvailabilityCheckWorker(self).limit_rate(response)
def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
uri, docname, lineno, status, info, code = result
filename = self.env.doc2path(docname, None)
linkstat = dict(filename=filename, lineno=lineno,
status=status, code=code, uri=uri,
info=info)
if status == 'unchecked':
self.write_linkstat(linkstat)
return
if status == 'working' and info == 'old':
self.write_linkstat(linkstat)
return
if lineno:
logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
if status == 'ignored':
if info:
logger.info(darkgray('-ignored- ') + uri + ': ' + info)
else:
logger.info(darkgray('-ignored- ') + uri)
self.write_linkstat(linkstat)
elif status == 'local':
logger.info(darkgray('-local- ') + uri)
self.write_entry('local', docname, filename, lineno, uri)
self.write_linkstat(linkstat)
elif status == 'working':
logger.info(darkgreen('ok ') + uri + info)
self.write_linkstat(linkstat)
elif status == 'broken':
if self.app.quiet or self.app.warningiserror:
logger.warning(__('broken link: %s (%s)'), uri, info,
location=(filename, lineno))
else:
logger.info(red('broken ') + uri + red(' - ' + info))
self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
self.write_linkstat(linkstat)
elif status == 'redirected':
try:
text, color = {
301: ('permanently', purple),
302: ('with Found', purple),
303: ('with See Other', purple),
307: ('temporarily', turquoise),
308: ('permanently', purple),
}[code]
except KeyError:
text, color = ('with unknown code', purple)
linkstat['text'] = text
logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info))
self.write_entry('redirected ' + text, docname, filename,
lineno, uri + ' to ' + info)
self.write_linkstat(linkstat)
else:
raise ValueError("Unknown status %s." % status)
def write_entry(self, what: str, docname: str, filename: str, line: int,
uri: str) -> None:
self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
def write_linkstat(self, data: dict) -> None:
self.json_outfile.write(json.dumps(data))
self.json_outfile.write('\n')
def finish(self) -> None:
logger.info('')
with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
total_links = 0
for hyperlink in self.hyperlinks.values():
if self.is_ignored_uri(hyperlink.uri):
self.process_result(
CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
'ignored', '', 0))
else:
self.wqueue.put(hyperlink, False)
total_links += 1
done = 0
while done < total_links:
self.process_result(self.rqueue.get())
done += 1
if self._broken:
self.app.statuscode = 1
self.wqueue.join()
# Shutdown threads.
for worker in self.workers:
self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
class HyperlinkAvailabilityCheckWorker(Thread):
"""A worker class for checking the availability of hyperlinks."""
def __init__(self, builder: CheckExternalLinksBuilder) -> None:
self.config = builder.config
self.env = builder.env
self.rate_limits = builder.rate_limits
self.rqueue = builder.rqueue
self.wqueue = builder.wqueue
self.anchors_ignore = [re.compile(x)
for x in self.config.linkcheck_anchors_ignore]
self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
in self.config.linkcheck_auth]
self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
self._good = builder._good
self._broken = builder._broken
self._redirected = builder._redirected
super().__init__(daemon=True)
def run(self) -> None:
kwargs = {} kwargs = {}
if self.config.linkcheck_timeout: if self.config.linkcheck_timeout:
kwargs['timeout'] = self.config.linkcheck_timeout kwargs['timeout'] = self.config.linkcheck_timeout
@ -372,96 +531,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
self.rate_limits[netloc] = RateLimit(delay, next_check) self.rate_limits[netloc] = RateLimit(delay, next_check)
return next_check return next_check
def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
uri, docname, lineno, status, info, code = result
filename = self.env.doc2path(docname, None)
linkstat = dict(filename=filename, lineno=lineno,
status=status, code=code, uri=uri,
info=info)
if status == 'unchecked':
self.write_linkstat(linkstat)
return
if status == 'working' and info == 'old':
self.write_linkstat(linkstat)
return
if lineno:
logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
if status == 'ignored':
if info:
logger.info(darkgray('-ignored- ') + uri + ': ' + info)
else:
logger.info(darkgray('-ignored- ') + uri)
self.write_linkstat(linkstat)
elif status == 'local':
logger.info(darkgray('-local- ') + uri)
self.write_entry('local', docname, filename, lineno, uri)
self.write_linkstat(linkstat)
elif status == 'working':
logger.info(darkgreen('ok ') + uri + info)
self.write_linkstat(linkstat)
elif status == 'broken':
if self.app.quiet or self.app.warningiserror:
logger.warning(__('broken link: %s (%s)'), uri, info,
location=(filename, lineno))
else:
logger.info(red('broken ') + uri + red(' - ' + info))
self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
self.write_linkstat(linkstat)
elif status == 'redirected':
try:
text, color = {
301: ('permanently', purple),
302: ('with Found', purple),
303: ('with See Other', purple),
307: ('temporarily', turquoise),
308: ('permanently', purple),
}[code]
except KeyError:
text, color = ('with unknown code', purple)
linkstat['text'] = text
logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info))
self.write_entry('redirected ' + text, docname, filename,
lineno, uri + ' to ' + info)
self.write_linkstat(linkstat)
else:
raise ValueError("Unknown status %s." % status)
def write_entry(self, what: str, docname: str, filename: str, line: int,
uri: str) -> None:
self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
def write_linkstat(self, data: dict) -> None:
self.json_outfile.write(json.dumps(data))
self.json_outfile.write('\n')
def finish(self) -> None:
logger.info('')
with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
total_links = 0
for hyperlink in self.hyperlinks.values():
if self.is_ignored_uri(hyperlink.uri):
self.process_result((hyperlink.uri, hyperlink.docname, hyperlink.lineno,
'ignored', '', 0))
else:
self.wqueue.put(hyperlink, False)
total_links += 1
done = 0
while done < total_links:
self.process_result(self.rqueue.get())
done += 1
if self._broken:
self.app.statuscode = 1
self.wqueue.join()
# Shutdown threads.
for worker in self.workers:
self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
class HyperlinkCollector(SphinxPostTransform): class HyperlinkCollector(SphinxPostTransform):
builders = ('linkcheck',) builders = ('linkcheck',)

View File

@ -47,12 +47,19 @@ class ApplicationError(SphinxError):
class ExtensionError(SphinxError): class ExtensionError(SphinxError):
"""Extension error.""" """Extension error."""
category = 'Extension error'
def __init__(self, message: str, orig_exc: Exception = None) -> None: def __init__(self, message: str, orig_exc: Exception = None, modname: str = None) -> None:
super().__init__(message) super().__init__(message)
self.message = message self.message = message
self.orig_exc = orig_exc self.orig_exc = orig_exc
self.modname = modname
@property
def category(self) -> str: # type: ignore
if self.modname:
return 'Extension error (%s)' % self.modname
else:
return 'Extension error'
def __repr__(self) -> str: def __repr__(self) -> str:
if self.orig_exc: if self.orig_exc:

View File

@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple, Tuple,
from sphinx.errors import ExtensionError, SphinxError from sphinx.errors import ExtensionError, SphinxError
from sphinx.locale import __ from sphinx.locale import __
from sphinx.util import logging from sphinx.util import logging
from sphinx.util.inspect import safe_getattr
if TYPE_CHECKING: if TYPE_CHECKING:
from sphinx.application import Sphinx from sphinx.application import Sphinx
@ -104,8 +105,9 @@ class EventManager:
except SphinxError: except SphinxError:
raise raise
except Exception as exc: except Exception as exc:
modname = safe_getattr(listener.handler, '__module__', None)
raise ExtensionError(__("Handler %r for event %r threw an exception") % raise ExtensionError(__("Handler %r for event %r threw an exception") %
(listener.handler, name), exc) from exc (listener.handler, name), exc, modname=modname) from exc
return results return results
def emit_firstresult(self, name: str, *args: Any, def emit_firstresult(self, name: str, *args: Any,

View File

@ -18,6 +18,17 @@ from sphinx.deprecation import RemovedInSphinx50Warning
FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding() FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
def getumask() -> int:
"""Get current umask value"""
umask = os.umask(0) # Note: Change umask value temporarily to obtain it
os.umask(umask)
return umask
UMASK = getumask()
class path(str): class path(str):
""" """
Represents a path which behaves like a string. Represents a path which behaves like a string.
@ -98,6 +109,16 @@ class path(str):
pointed to by the symbolic links are copied. pointed to by the symbolic links are copied.
""" """
shutil.copytree(self, destination, symlinks=symlinks) shutil.copytree(self, destination, symlinks=symlinks)
if os.environ.get('SPHINX_READONLY_TESTDIR'):
# If source tree is marked read-only (e.g. because it is on a read-only
# filesystem), `shutil.copytree` will mark the destination as read-only
# as well. To avoid failures when adding additional files/directories
# to the destination tree, ensure destination directories are not marked
# read-only.
for root, dirs, files in os.walk(destination):
os.chmod(root, 0o755 & ~UMASK)
for name in files:
os.chmod(os.path.join(root, name), 0o644 & ~UMASK)
def movetree(self, destination: str) -> None: def movetree(self, destination: str) -> None:
""" """

View File

@ -173,7 +173,7 @@
% FIXME: this is unrelated to an option, move this elsewhere % FIXME: this is unrelated to an option, move this elsewhere
% To allow hyphenation of first word in narrow contexts; no option, % To allow hyphenation of first word in narrow contexts; no option,
% customization to be done via 'preamble' key % customization to be done via 'preamble' key
\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip} \newcommand*\sphinxAtStartPar{\hskip\z@skip}
% No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex % No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex
\ifdefined\directlua\let\sphinxAtStartPar\@empty\fi \ifdefined\directlua\let\sphinxAtStartPar\@empty\fi
% user interface: options can be changed midway in a document! % user interface: options can be changed midway in a document!

View File

@ -21,7 +21,8 @@ from unittest import mock
import pytest import pytest
import requests import requests
from sphinx.builders.linkcheck import CheckExternalLinksBuilder, RateLimit from sphinx.builders.linkcheck import (CheckExternalLinksBuilder,
HyperlinkAvailabilityCheckWorker, RateLimit)
from sphinx.util.console import strip_colors from sphinx.util.console import strip_colors
from .utils import CERT_FILE, http_server, https_server from .utils import CERT_FILE, http_server, https_server
@ -536,40 +537,50 @@ class FakeResponse:
def test_limit_rate_default_sleep(app): def test_limit_rate_default_sleep(app):
checker = CheckExternalLinksBuilder(app) checker = CheckExternalLinksBuilder(app)
checker.init()
checker.rate_limits = {} checker.rate_limits = {}
worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0): with mock.patch('time.time', return_value=0.0):
next_check = checker.limit_rate(FakeResponse()) next_check = worker.limit_rate(FakeResponse())
assert next_check == 60.0 assert next_check == 60.0
def test_limit_rate_user_max_delay(app): def test_limit_rate_user_max_delay(app):
app.config.linkcheck_rate_limit_timeout = 0.0 app.config.linkcheck_rate_limit_timeout = 0.0
checker = CheckExternalLinksBuilder(app) checker = CheckExternalLinksBuilder(app)
checker.init()
checker.rate_limits = {} checker.rate_limits = {}
next_check = checker.limit_rate(FakeResponse()) worker = HyperlinkAvailabilityCheckWorker(checker)
next_check = worker.limit_rate(FakeResponse())
assert next_check is None assert next_check is None
def test_limit_rate_doubles_previous_wait_time(app): def test_limit_rate_doubles_previous_wait_time(app):
checker = CheckExternalLinksBuilder(app) checker = CheckExternalLinksBuilder(app)
checker.init()
checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)} checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0): with mock.patch('time.time', return_value=0.0):
next_check = checker.limit_rate(FakeResponse()) next_check = worker.limit_rate(FakeResponse())
assert next_check == 120.0 assert next_check == 120.0
def test_limit_rate_clips_wait_time_to_max_time(app): def test_limit_rate_clips_wait_time_to_max_time(app):
checker = CheckExternalLinksBuilder(app) checker = CheckExternalLinksBuilder(app)
checker.init()
app.config.linkcheck_rate_limit_timeout = 90.0 app.config.linkcheck_rate_limit_timeout = 90.0
checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)} checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0): with mock.patch('time.time', return_value=0.0):
next_check = checker.limit_rate(FakeResponse()) next_check = worker.limit_rate(FakeResponse())
assert next_check == 90.0 assert next_check == 90.0
def test_limit_rate_bails_out_after_waiting_max_time(app): def test_limit_rate_bails_out_after_waiting_max_time(app):
checker = CheckExternalLinksBuilder(app) checker = CheckExternalLinksBuilder(app)
checker.init()
app.config.linkcheck_rate_limit_timeout = 90.0 app.config.linkcheck_rate_limit_timeout = 90.0
checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)} checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)}
next_check = checker.limit_rate(FakeResponse()) worker = HyperlinkAvailabilityCheckWorker(checker)
next_check = worker.limit_rate(FakeResponse())
assert next_check is None assert next_check is None