From a9b0f2708b67a355c5f4969ccbbe9bd695518805 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 4 Jan 2023 04:22:20 +0000 Subject: [PATCH] Cache doctrees more efficiently --- sphinx/environment/__init__.py | 23 ++++++++++++++--------- sphinx/environment/adapters/toctree.py | 2 +- sphinx/testing/util.py | 4 ++++ tests/test_build_gettext.py | 3 ++- tests/test_build_html.py | 2 +- tests/test_build_latex.py | 2 +- tests/test_ext_math.py | 4 ++-- 7 files changed, 25 insertions(+), 15 deletions(-) diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index b04af59f7..1f279ceb1 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -6,7 +6,7 @@ import functools import os import pickle from collections import defaultdict -from copy import copy, deepcopy +from copy import copy from datetime import datetime from os import path from typing import TYPE_CHECKING, Any, Callable, Generator, Iterator @@ -178,6 +178,9 @@ class BuildEnvironment: # docnames to re-read unconditionally on next build self.reread_always: set[str] = set() + # docname -> pickled doctree + self._pickled_doctree_cache: dict[str, bytes] = {} + # File metadata # docname -> dict of metadata items self.metadata: dict[str, dict[str, Any]] = defaultdict(dict) @@ -577,20 +580,22 @@ class BuildEnvironment: def get_doctree(self, docname: str) -> nodes.document: """Read the doctree for a file from the pickle and return it.""" - doctreedir = self.doctreedir - - @functools.lru_cache(maxsize=None) - def _load_doctree_from_disk(docname: str) -> nodes.document: - """Read the doctree for a file from the pickle and return it.""" - filename = path.join(doctreedir, docname + '.doctree') + try: + serialised = self._pickled_doctree_cache[docname] + except KeyError: + filename = path.join(self.doctreedir, docname + '.doctree') with open(filename, 'rb') as f: - return pickle.load(f) + serialised = self._pickled_doctree_cache[docname] = f.read() - doctree = deepcopy(_load_doctree_from_disk(docname)) + doctree = pickle.loads(serialised) doctree.settings.env = self doctree.reporter = LoggingReporter(self.doc2path(docname)) return doctree + @functools.cached_property + def master_doctree(self) -> nodes.document: + return self.get_doctree(self.config.root_doc) + def get_and_resolve_doctree( self, docname: str, diff --git a/sphinx/environment/adapters/toctree.py b/sphinx/environment/adapters/toctree.py index d646fa2cc..a2299c469 100644 --- a/sphinx/environment/adapters/toctree.py +++ b/sphinx/environment/adapters/toctree.py @@ -319,7 +319,7 @@ class TocTree: def get_toctree_for(self, docname: str, builder: Builder, collapse: bool, **kwargs: Any) -> Element | None: """Return the global TOC nodetree.""" - doctree = self.env.get_doctree(self.env.config.root_doc) + doctree = self.env.master_doctree toctrees: list[Element] = [] if 'includehidden' not in kwargs: kwargs['includehidden'] = True diff --git a/sphinx/testing/util.py b/sphinx/testing/util.py index 79d1390aa..3568e3723 100644 --- a/sphinx/testing/util.py +++ b/sphinx/testing/util.py @@ -156,6 +156,10 @@ class SphinxTestApp(application.Sphinx): def __repr__(self) -> str: return f'<{self.__class__.__name__} buildername={self.builder.name!r}>' + def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None: + self.env._pickled_doctree_cache.clear() + super().build(force_all, filenames) + class SphinxTestAppWrapperForSkipBuilding: """ diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 630f0760c..a4551ad53 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -135,6 +135,7 @@ def test_gettext_index_entries(app): 'gettext_additional_targets': []}) def test_gettext_disable_index_entries(app): # regression test for #976 + app.env._pickled_doctree_cache.clear() # clear cache app.builder.build(['index_entries']) _msgid_getter = re.compile(r'msgid "(.*)"').search @@ -165,7 +166,7 @@ def test_gettext_disable_index_entries(app): @pytest.mark.sphinx('gettext', testroot='intl', srcdir='gettext') def test_gettext_template(app): - app.builder.build_all() + app.build() assert (app.outdir / 'sphinx.pot').isfile() result = (app.outdir / 'sphinx.pot').read_text(encoding='utf8') diff --git a/tests/test_build_html.py b/tests/test_build_html.py index 5b27a4796..5eed5d5ea 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -665,7 +665,7 @@ def test_numfig_without_numbered_toctree_warn(app, warning): index = (app.srcdir / 'index.rst').read_text(encoding='utf8') index = re.sub(':numbered:.*', '', index) (app.srcdir / 'index.rst').write_text(index, encoding='utf8') - app.builder.build_all() + app.build() warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 35947d372..f20fc67c5 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -429,7 +429,7 @@ def test_numref_with_prefix2(app, status, warning): 'latex', testroot='numfig', confoverrides={'numfig': True, 'language': 'ja'}) def test_numref_with_language_ja(app, status, warning): - app.builder.build_all() + app.build() result = (app.outdir / 'python.tex').read_text(encoding='utf8') print(result) print(status.getvalue()) diff --git a/tests/test_ext_math.py b/tests/test_ext_math.py index 206c36ca9..cd49b5244 100644 --- a/tests/test_ext_math.py +++ b/tests/test_ext_math.py @@ -108,7 +108,7 @@ def test_mathjax_align(app, status, warning): confoverrides={'math_number_all': True, 'extensions': ['sphinx.ext.mathjax']}) def test_math_number_all_mathjax(app, status, warning): - app.builder.build_all() + app.build() content = (app.outdir / 'index.html').read_text(encoding='utf8') html = (r'
\s*' @@ -119,7 +119,7 @@ def test_math_number_all_mathjax(app, status, warning): @pytest.mark.sphinx('latex', testroot='ext-math', confoverrides={'extensions': ['sphinx.ext.mathjax']}) def test_math_number_all_latex(app, status, warning): - app.builder.build_all() + app.build() content = (app.outdir / 'python.tex').read_text(encoding='utf8') macro = (r'\\begin{equation\*}\s*'