IntenseWebs/sphinx
3
0
Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2025-02-25 18:55:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.x'

Browse Source
This commit is contained in:
Takeshi KOMIYA 2021-02-07 17:01:09 +09:00
commit 84458da828
9 changed files with 239 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES
View File

@ -99,9 +99,12 @@ Deprecated
----------
* pending_xref node for viewcode extension
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected``
* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
* ``sphinx.builders.linkcheck.node_line_or_0()``
* ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()``
* ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter``
@ -148,6 +151,7 @@ Features added
dedent via no-argument ``:dedent:`` option
* C++, also hyperlink operator overloads in expressions and alias declarations.
* #8247: Allow production lists to refer to tokens from other production groups
* #8813: Show what extension (or module) caused it on errors on event handler
Bugs fixed
----------

2
doc/_themes/sphinx13/layout.html vendored
View File

@ -67,7 +67,7 @@
<li><a href="{{ pathto('index') }}">Home</a></li>
<li><a href="{{ pathto('usage/installation') }}">Get it</a></li>
<li><a href="{{ pathto('contents') }}">Docs</a></li>
<li><a href="{{ pathto('development') }}">Extend</a></li>
<li><a href="{{ pathto('development/index') }}">Extend</a></li>
</ul>
<div>
<a href="{{ pathto('index') }}">

15
doc/extdev/deprecated.rst
View File

@ -72,6 +72,16 @@ The following is a list of deprecated interfaces.
- 5.0
- ``sphinx.ext.viewcode.viewcode_anchor``
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
- 3.5
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
- 3.5
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
- 3.5
- 5.0
@ -87,6 +97,11 @@ The following is a list of deprecated interfaces.
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
- 3.5
- 5.0
- N/A
* - ``sphinx.builders.linkcheck.node_line_or_0()``
- 3.5
- 5.0

267
sphinx/builders/linkcheck.py
View File

@ -12,14 +12,14 @@ import json
import queue
import re
import socket
import threading
import time
import warnings
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from html.parser import HTMLParser
from os import path
from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast
from threading import Thread
from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple, cast
from urllib.parse import unquote, urlparse
from docutils import nodes
@ -44,6 +44,12 @@ Hyperlink = NamedTuple('Hyperlink', (('next_check', float),
('uri', Optional[str]),
('docname', Optional[str]),
('lineno', Optional[int])))
CheckResult = NamedTuple('CheckResult', (('uri', str),
('docname', str),
('lineno', int),
('status', str),
('message', str),
('code', int)))
RateLimit = NamedTuple('RateLimit', (('delay', float), ('next_check', float)))
DEFAULT_REQUEST_HEADERS = {
@ -108,11 +114,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
def init(self) -> None:
self.hyperlinks = {} # type: Dict[str, Hyperlink]
self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
self.anchors_ignore = [re.compile(x)
for x in self.config.linkcheck_anchors_ignore]
self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
in self.config.linkcheck_auth]
self._good = set() # type: Set[str]
self._broken = {} # type: Dict[str, str]
self._redirected = {} # type: Dict[str, Tuple[str, int]]
@ -123,15 +124,43 @@ class CheckExternalLinksBuilder(DummyBuilder):
self.rate_limits = {} # type: Dict[str, RateLimit]
self.wqueue = queue.PriorityQueue() # type: queue.PriorityQueue
self.rqueue = queue.Queue() # type: queue.Queue
self.workers = [] # type: List[threading.Thread]
self.workers = [] # type: List[Thread]
for i in range(self.config.linkcheck_workers):
thread = threading.Thread(target=self.check_thread, daemon=True)
thread = HyperlinkAvailabilityCheckWorker(self)
thread.start()
self.workers.append(thread)
def is_ignored_uri(self, uri: str) -> bool:
return any(pat.match(uri) for pat in self.to_ignore)
@property
def anchors_ignore(self) -> List[Pattern]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "anchors_ignore"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return [re.compile(x) for x in self.config.linkcheck_anchors_ignore]
@property
def auth(self) -> List[Tuple[Pattern, Any]]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "auth"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return [(re.compile(pattern), auth_info) for pattern, auth_info
in self.config.linkcheck_auth]
@property
def to_ignore(self) -> List[Pattern]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "to_ignore"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return [re.compile(x) for x in self.config.linkcheck_ignore]
@property
def good(self) -> Set[str]:
warnings.warn(
@ -160,6 +189,136 @@ class CheckExternalLinksBuilder(DummyBuilder):
return self._redirected
def check_thread(self) -> None:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "check_thread"),
RemovedInSphinx50Warning,
stacklevel=2,
)
# do nothing.
def limit_rate(self, response: Response) -> Optional[float]:
warnings.warn(
"%s.%s is deprecated." % (self.__class__.__name__, "limit_rate"),
RemovedInSphinx50Warning,
stacklevel=2,
)
return HyperlinkAvailabilityCheckWorker(self).limit_rate(response)
def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
uri, docname, lineno, status, info, code = result
filename = self.env.doc2path(docname, None)
linkstat = dict(filename=filename, lineno=lineno,
status=status, code=code, uri=uri,
info=info)
if status == 'unchecked':
self.write_linkstat(linkstat)
return
if status == 'working' and info == 'old':
self.write_linkstat(linkstat)
return
if lineno:
logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
if status == 'ignored':
if info:
logger.info(darkgray('-ignored- ') + uri + ': ' + info)
else:
logger.info(darkgray('-ignored- ') + uri)
self.write_linkstat(linkstat)
elif status == 'local':
logger.info(darkgray('-local- ') + uri)
self.write_entry('local', docname, filename, lineno, uri)
self.write_linkstat(linkstat)
elif status == 'working':
logger.info(darkgreen('ok ') + uri + info)
self.write_linkstat(linkstat)
elif status == 'broken':
if self.app.quiet or self.app.warningiserror:
logger.warning(__('broken link: %s (%s)'), uri, info,
location=(filename, lineno))
else:
logger.info(red('broken ') + uri + red(' - ' + info))
self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
self.write_linkstat(linkstat)
elif status == 'redirected':
try:
text, color = {
301: ('permanently', purple),
302: ('with Found', purple),
303: ('with See Other', purple),
307: ('temporarily', turquoise),
308: ('permanently', purple),
}[code]
except KeyError:
text, color = ('with unknown code', purple)
linkstat['text'] = text
logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info))
self.write_entry('redirected ' + text, docname, filename,
lineno, uri + ' to ' + info)
self.write_linkstat(linkstat)
else:
raise ValueError("Unknown status %s." % status)
def write_entry(self, what: str, docname: str, filename: str, line: int,
uri: str) -> None:
self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
def write_linkstat(self, data: dict) -> None:
self.json_outfile.write(json.dumps(data))
self.json_outfile.write('\n')
def finish(self) -> None:
logger.info('')
with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
total_links = 0
for hyperlink in self.hyperlinks.values():
if self.is_ignored_uri(hyperlink.uri):
self.process_result(
CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
'ignored', '', 0))
else:
self.wqueue.put(hyperlink, False)
total_links += 1
done = 0
while done < total_links:
self.process_result(self.rqueue.get())
done += 1
if self._broken:
self.app.statuscode = 1
self.wqueue.join()
# Shutdown threads.
for worker in self.workers:
self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
class HyperlinkAvailabilityCheckWorker(Thread):
"""A worker class for checking the availability of hyperlinks."""
def __init__(self, builder: CheckExternalLinksBuilder) -> None:
self.config = builder.config
self.env = builder.env
self.rate_limits = builder.rate_limits
self.rqueue = builder.rqueue
self.wqueue = builder.wqueue
self.anchors_ignore = [re.compile(x)
for x in self.config.linkcheck_anchors_ignore]
self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
in self.config.linkcheck_auth]
self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
self._good = builder._good
self._broken = builder._broken
self._redirected = builder._redirected
super().__init__(daemon=True)
def run(self) -> None:
kwargs = {}
if self.config.linkcheck_timeout:
kwargs['timeout'] = self.config.linkcheck_timeout
@ -372,96 +531,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
self.rate_limits[netloc] = RateLimit(delay, next_check)
return next_check
def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
uri, docname, lineno, status, info, code = result
filename = self.env.doc2path(docname, None)
linkstat = dict(filename=filename, lineno=lineno,
status=status, code=code, uri=uri,
info=info)
if status == 'unchecked':
self.write_linkstat(linkstat)
return
if status == 'working' and info == 'old':
self.write_linkstat(linkstat)
return
if lineno:
logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
if status == 'ignored':
if info:
logger.info(darkgray('-ignored- ') + uri + ': ' + info)
else:
logger.info(darkgray('-ignored- ') + uri)
self.write_linkstat(linkstat)
elif status == 'local':
logger.info(darkgray('-local- ') + uri)
self.write_entry('local', docname, filename, lineno, uri)
self.write_linkstat(linkstat)
elif status == 'working':
logger.info(darkgreen('ok ') + uri + info)
self.write_linkstat(linkstat)
elif status == 'broken':
if self.app.quiet or self.app.warningiserror:
logger.warning(__('broken link: %s (%s)'), uri, info,
location=(filename, lineno))
else:
logger.info(red('broken ') + uri + red(' - ' + info))
self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
self.write_linkstat(linkstat)
elif status == 'redirected':
try:
text, color = {
301: ('permanently', purple),
302: ('with Found', purple),
303: ('with See Other', purple),
307: ('temporarily', turquoise),
308: ('permanently', purple),
}[code]
except KeyError:
text, color = ('with unknown code', purple)
linkstat['text'] = text
logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info))
self.write_entry('redirected ' + text, docname, filename,
lineno, uri + ' to ' + info)
self.write_linkstat(linkstat)
else:
raise ValueError("Unknown status %s." % status)
def write_entry(self, what: str, docname: str, filename: str, line: int,
uri: str) -> None:
self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
def write_linkstat(self, data: dict) -> None:
self.json_outfile.write(json.dumps(data))
self.json_outfile.write('\n')
def finish(self) -> None:
logger.info('')
with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
total_links = 0
for hyperlink in self.hyperlinks.values():
if self.is_ignored_uri(hyperlink.uri):
self.process_result((hyperlink.uri, hyperlink.docname, hyperlink.lineno,
'ignored', '', 0))
else:
self.wqueue.put(hyperlink, False)
total_links += 1
done = 0
while done < total_links:
self.process_result(self.rqueue.get())
done += 1
if self._broken:
self.app.statuscode = 1
self.wqueue.join()
# Shutdown threads.
for worker in self.workers:
self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
class HyperlinkCollector(SphinxPostTransform):
builders = ('linkcheck',)

11
sphinx/errors.py
View File

@ -47,12 +47,19 @@ class ApplicationError(SphinxError):
class ExtensionError(SphinxError):
"""Extension error."""
category = 'Extension error'
def __init__(self, message: str, orig_exc: Exception = None) -> None:
def __init__(self, message: str, orig_exc: Exception = None, modname: str = None) -> None:
super().__init__(message)
self.message = message
self.orig_exc = orig_exc
self.modname = modname
@property
def category(self) -> str: # type: ignore
if self.modname:
return 'Extension error (%s)' % self.modname
else:
return 'Extension error'
def __repr__(self) -> str:
if self.orig_exc:

4
sphinx/events.py
View File

@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple, Tuple,
from sphinx.errors import ExtensionError, SphinxError
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.inspect import safe_getattr
if TYPE_CHECKING:
from sphinx.application import Sphinx
@ -104,8 +105,9 @@ class EventManager:
except SphinxError:
raise
except Exception as exc:
modname = safe_getattr(listener.handler, '__module__', None)
raise ExtensionError(__("Handler %r for event %r threw an exception") %
(listener.handler, name), exc) from exc
(listener.handler, name), exc, modname=modname) from exc
return results
def emit_firstresult(self, name: str, *args: Any,

21
sphinx/testing/path.py
View File

@ -18,6 +18,17 @@ from sphinx.deprecation import RemovedInSphinx50Warning
FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
def getumask() -> int:
"""Get current umask value"""
umask = os.umask(0) # Note: Change umask value temporarily to obtain it
os.umask(umask)
return umask
UMASK = getumask()
class path(str):
"""
Represents a path which behaves like a string.
@ -98,6 +109,16 @@ class path(str):
pointed to by the symbolic links are copied.
"""
shutil.copytree(self, destination, symlinks=symlinks)
if os.environ.get('SPHINX_READONLY_TESTDIR'):
# If source tree is marked read-only (e.g. because it is on a read-only
# filesystem), `shutil.copytree` will mark the destination as read-only
# as well. To avoid failures when adding additional files/directories
# to the destination tree, ensure destination directories are not marked
# read-only.
for root, dirs, files in os.walk(destination):
os.chmod(root, 0o755 & ~UMASK)
for name in files:
os.chmod(os.path.join(root, name), 0o644 & ~UMASK)
def movetree(self, destination: str) -> None:
"""

2
sphinx/texinputs/sphinx.sty
View File

@ -173,7 +173,7 @@
% FIXME: this is unrelated to an option, move this elsewhere
% To allow hyphenation of first word in narrow contexts; no option,
% customization to be done via 'preamble' key
\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip}
\newcommand*\sphinxAtStartPar{\hskip\z@skip}
% No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex
\ifdefined\directlua\let\sphinxAtStartPar\@empty\fi
% user interface: options can be changed midway in a document!

23
tests/test_build_linkcheck.py
View File

@ -21,7 +21,8 @@ from unittest import mock
import pytest
import requests
from sphinx.builders.linkcheck import CheckExternalLinksBuilder, RateLimit
from sphinx.builders.linkcheck import (CheckExternalLinksBuilder,
HyperlinkAvailabilityCheckWorker, RateLimit)
from sphinx.util.console import strip_colors
from .utils import CERT_FILE, http_server, https_server
@ -536,40 +537,50 @@ class FakeResponse:
def test_limit_rate_default_sleep(app):
checker = CheckExternalLinksBuilder(app)
checker.init()
checker.rate_limits = {}
worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0):
next_check = checker.limit_rate(FakeResponse())
next_check = worker.limit_rate(FakeResponse())
assert next_check == 60.0
def test_limit_rate_user_max_delay(app):
app.config.linkcheck_rate_limit_timeout = 0.0
checker = CheckExternalLinksBuilder(app)
checker.init()
checker.rate_limits = {}
next_check = checker.limit_rate(FakeResponse())
worker = HyperlinkAvailabilityCheckWorker(checker)
next_check = worker.limit_rate(FakeResponse())
assert next_check is None
def test_limit_rate_doubles_previous_wait_time(app):
checker = CheckExternalLinksBuilder(app)
checker.init()
checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0):
next_check = checker.limit_rate(FakeResponse())
next_check = worker.limit_rate(FakeResponse())
assert next_check == 120.0
def test_limit_rate_clips_wait_time_to_max_time(app):
checker = CheckExternalLinksBuilder(app)
checker.init()
app.config.linkcheck_rate_limit_timeout = 90.0
checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
worker = HyperlinkAvailabilityCheckWorker(checker)
with mock.patch('time.time', return_value=0.0):
next_check = checker.limit_rate(FakeResponse())
next_check = worker.limit_rate(FakeResponse())
assert next_check == 90.0
def test_limit_rate_bails_out_after_waiting_max_time(app):
checker = CheckExternalLinksBuilder(app)
checker.init()
app.config.linkcheck_rate_limit_timeout = 90.0
checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)}
next_check = checker.limit_rate(FakeResponse())
worker = HyperlinkAvailabilityCheckWorker(checker)
next_check = worker.limit_rate(FakeResponse())
assert next_check is None