From 0e2cb4793e97bd7736ae6742c15ca5eb1afa497c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 10 May 2018 00:43:48 +0900 Subject: [PATCH 1/4] Make Config picklable --- sphinx/config.py | 34 +++++++++++++++++++++++++++++++++- sphinx/environment/__init__.py | 13 +------------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/sphinx/config.py b/sphinx/config.py index 58e5be277..0a2b184c0 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -11,12 +11,15 @@ import re import traceback +import types import warnings from collections import OrderedDict from os import path, getenv from typing import Any, NamedTuple, Union -from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types +from six import ( + PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types, class_types +) from sphinx.deprecation import RemovedInSphinx30Warning from sphinx.errors import ConfigError, ExtensionError @@ -35,6 +38,7 @@ if False: logger = logging.getLogger(__name__) CONFIG_FILENAME = 'conf.py' +UNSERIALIZEABLE_TYPES = class_types + (types.ModuleType, types.FunctionType) copyright_year_re = re.compile(r'^((\d{4}-)?)(\d{4})(?=[ ,])') if PY3: @@ -308,6 +312,34 @@ class Config(object): rebuild = [rebuild] return (value for value in self if value.rebuild in rebuild) + def __getstate__(self): + # type: () -> Dict + """Obtains serializable data for pickling.""" + # remove potentially pickling-problematic values from config + __dict__ = {} + for key, value in iteritems(self.__dict__): + if key.startswith('_') or isinstance(value, UNSERIALIZEABLE_TYPES): + pass + else: + __dict__[key] = value + + # create a picklable copy of values list + __dict__['values'] = {} + for key, value in iteritems(self.values): # type: ignore + real_value = getattr(self, key) + if isinstance(real_value, UNSERIALIZEABLE_TYPES): + # omit unserializable value + real_value = None + + # types column is also omitted + __dict__['values'][key] = (real_value, value[1], None) + + return __dict__ + + def __setstate__(self, state): + # type: (Dict) -> None + self.__dict__.update(state) + def eval_config_file(filename, tags): # type: (unicode, Tags) -> Dict[unicode, Any] diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index e8d9dd9d4..6a3951e9d 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -12,14 +12,13 @@ import os import re import sys -import types import warnings from collections import defaultdict from copy import copy from os import path from docutils.utils import Reporter, get_source_line -from six import BytesIO, class_types, next +from six import BytesIO, next from six.moves import cPickle as pickle from sphinx import addnodes @@ -126,21 +125,11 @@ class BuildEnvironment(object): # remove unpicklable attributes app = env.app del env.app - values = env.config.values - del env.config.values domains = env.domains del env.domains - # remove potentially pickling-problematic values from config - for key, val in list(vars(env.config).items()): - if key.startswith('_') or \ - isinstance(val, types.ModuleType) or \ - isinstance(val, types.FunctionType) or \ - isinstance(val, class_types): - del env.config[key] pickle.dump(env, f, pickle.HIGHEST_PROTOCOL) # reset attributes env.domains = domains - env.config.values = values env.app = app @classmethod From 8e2225b4fce2957a617bb1d9e05a6e51788ee636 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 10 May 2018 00:59:53 +0900 Subject: [PATCH 2/4] Make BuildEnvironment picklable directly --- sphinx/environment/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index 6a3951e9d..554e3b033 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -122,15 +122,7 @@ class BuildEnvironment(object): @staticmethod def dump(env, f): # type: (BuildEnvironment, IO) -> None - # remove unpicklable attributes - app = env.app - del env.app - domains = env.domains - del env.domains pickle.dump(env, f, pickle.HIGHEST_PROTOCOL) - # reset attributes - env.domains = domains - env.app = app @classmethod def dumps(cls, env): @@ -246,6 +238,17 @@ class BuildEnvironment(object): # attributes of "any" cross references self.ref_context = {} # type: Dict[unicode, Any] + def __getstate__(self): + # type: () -> Dict + """Obtains serializable data for pickling.""" + __dict__ = self.__dict__.copy() + __dict__.update(app=None, domains={}) # clear unpickable attributes + return __dict__ + + def __setstate__(self, state): + # type: (Dict) -> None + self.__dict__.update(state) + def set_warnfunc(self, func): # type: (Callable) -> None warnings.warn('env.set_warnfunc() is now deprecated. Use sphinx.util.logging instead.', From 48a194159149c23618e51960e22f19d2883c27ef Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 10 May 2018 01:04:30 +0900 Subject: [PATCH 3/4] Increment ENV_VERSION --- sphinx/environment/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index 554e3b033..979846ab8 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -67,7 +67,7 @@ default_settings = { # or changed to properly invalidate pickle files. # # NOTE: increase base version by 2 to have distinct numbers for Py2 and 3 -ENV_VERSION = 52 + (sys.version_info[0] - 2) +ENV_VERSION = 53 + (sys.version_info[0] - 2) versioning_conditions = { From d6db20781a2b1a580a4014e82e232ca2ec387761 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 10 May 2018 01:25:22 +0900 Subject: [PATCH 4/4] Deprecate methods for pickling/unpickling on BuildEnvironment --- CHANGES | 6 ++ doc/extdev/index.rst | 30 ++++++++++ sphinx/application.py | 6 +- sphinx/builders/__init__.py | 12 ++-- sphinx/environment/__init__.py | 106 +++++++++++++++++++-------------- 5 files changed, 108 insertions(+), 52 deletions(-) diff --git a/CHANGES b/CHANGES index 60a0c6f4f..0b200f57f 100644 --- a/CHANGES +++ b/CHANGES @@ -62,6 +62,12 @@ Deprecated * ``sphinx.writers.latex.LaTeXWriter.restrict_footnote()`` is deprecated * ``sphinx.writers.latex.LaTeXWriter.unrestrict_footnote()`` is deprecated * ``LaTeXWriter.bibitems`` is deprecated +* ``BuildEnvironment.load()`` is deprecated +* ``BuildEnvironment.loads()`` is deprecated +* ``BuildEnvironment.frompickle()`` is deprecated +* ``BuildEnvironment.dump()`` is deprecated +* ``BuildEnvironment.dumps()`` is deprecated +* ``BuildEnvironment.topickle()`` is deprecated For more details, see `deprecation APIs list `_ diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst index dfe8dbaf0..5f480b51f 100644 --- a/doc/extdev/index.rst +++ b/doc/extdev/index.rst @@ -192,6 +192,36 @@ The following is a list of deprecated interface. - 3.0 - :meth:`~sphinx.application.Sphinx.add_domain()` + * - ``BuildEnvironment.load()`` + - 1.8 + - 3.0 + - ``pickle.load()`` + + * - ``BuildEnvironment.loads()`` + - 1.8 + - 3.0 + - ``pickle.loads()`` + + * - ``BuildEnvironment.frompickle()`` + - 1.8 + - 3.0 + - ``pickle.load()`` + + * - ``BuildEnvironment.dump()`` + - 1.8 + - 3.0 + - ``pickle.dump()`` + + * - ``BuildEnvironment.dumps()`` + - 1.8 + - 3.0 + - ``pickle.dumps()`` + + * - ``BuildEnvironment.topickle()`` + - 1.8 + - 3.0 + - ``pickle.dump()`` + * - ``BuildEnvironment._nitpick_ignore`` - 1.8 - 3.0 diff --git a/sphinx/application.py b/sphinx/application.py index 97b9944d2..8e21d837a 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -22,6 +22,7 @@ from os import path from docutils.parsers.rst import Directive, directives, roles from six import itervalues +from six.moves import cPickle as pickle from six.moves import cStringIO import sphinx @@ -291,7 +292,10 @@ class Sphinx(object): else: try: logger.info(bold(__('loading pickled environment... ')), nonl=True) - self.env = BuildEnvironment.frompickle(filename, self) + with open(filename, 'rb') as f: + self.env = pickle.load(f) + self.env.app = self + self.env.config.values = self.config.values needed, reason = self.env.need_refresh(self) if needed: raise IOError(reason) diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 841489704..b44139a55 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -17,7 +17,6 @@ from docutils import nodes from six.moves import cPickle as pickle from sphinx.deprecation import RemovedInSphinx20Warning -from sphinx.environment import BuildEnvironment from sphinx.environment.adapters.asset import ImageAdapter from sphinx.errors import SphinxError from sphinx.io import read_doc @@ -372,7 +371,8 @@ class Builder(object): # save the environment from sphinx.application import ENV_PICKLE_FILENAME logger.info(bold(__('pickling environment... ')), nonl=True) - self.env.topickle(path.join(self.doctreedir, ENV_PICKLE_FILENAME)) + with open(path.join(self.doctreedir, ENV_PICKLE_FILENAME), 'wb') as f: + pickle.dump(self.env, f, pickle.HIGHEST_PROTOCOL) logger.info(__('done')) # global actions @@ -492,16 +492,16 @@ class Builder(object): self.env.clear_doc(docname) def read_process(docs): - # type: (List[unicode]) -> unicode + # type: (List[unicode]) -> bytes self.env.app = self.app for docname in docs: self.read_doc(docname) # allow pickling self to send it back - return BuildEnvironment.dumps(self.env) + return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL) def merge(docs, otherenv): - # type: (List[unicode], unicode) -> None - env = BuildEnvironment.loads(otherenv) + # type: (List[unicode], bytes) -> None + env = pickle.loads(otherenv) self.env.merge_info_from(docs, env, self.app) tasks = ParallelTasks(nproc) diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index 979846ab8..3e65e15a6 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -91,51 +91,6 @@ class BuildEnvironment(object): domains = None # type: Dict[unicode, Domain] - # --------- ENVIRONMENT PERSISTENCE ---------------------------------------- - - @staticmethod - def load(f, app=None): - # type: (IO, Sphinx) -> BuildEnvironment - try: - env = pickle.load(f) - except Exception as exc: - # This can happen for example when the pickle is from a - # different version of Sphinx. - raise IOError(exc) - if app: - env.app = app - env.config.values = app.config.values - return env - - @classmethod - def loads(cls, string, app=None): - # type: (unicode, Sphinx) -> BuildEnvironment - io = BytesIO(string) - return cls.load(io, app) - - @classmethod - def frompickle(cls, filename, app): - # type: (unicode, Sphinx) -> BuildEnvironment - with open(filename, 'rb') as f: - return cls.load(f, app) - - @staticmethod - def dump(env, f): - # type: (BuildEnvironment, IO) -> None - pickle.dump(env, f, pickle.HIGHEST_PROTOCOL) - - @classmethod - def dumps(cls, env): - # type: (BuildEnvironment) -> unicode - io = BytesIO() - cls.dump(env, io) - return io.getvalue() - - def topickle(self, filename): - # type: (unicode) -> None - with open(filename, 'wb') as f: - self.dump(self, f) - # --------- ENVIRONMENT INITIALIZATION ------------------------------------- def __init__(self, app): @@ -816,3 +771,64 @@ class BuildEnvironment(object): 'Please use config.nitpick_ignore instead.', RemovedInSphinx30Warning) return self.config.nitpick_ignore + + @staticmethod + def load(f, app=None): + # type: (IO, Sphinx) -> BuildEnvironment + warnings.warn('BuildEnvironment.load() is deprecated. ' + 'Please use pickle.load() instead.', + RemovedInSphinx30Warning) + try: + env = pickle.load(f) + except Exception as exc: + # This can happen for example when the pickle is from a + # different version of Sphinx. + raise IOError(exc) + if app: + env.app = app + env.config.values = app.config.values + return env + + @classmethod + def loads(cls, string, app=None): + # type: (unicode, Sphinx) -> BuildEnvironment + warnings.warn('BuildEnvironment.loads() is deprecated. ' + 'Please use pickle.loads() instead.', + RemovedInSphinx30Warning) + io = BytesIO(string) + return cls.load(io, app) + + @classmethod + def frompickle(cls, filename, app): + # type: (unicode, Sphinx) -> BuildEnvironment + warnings.warn('BuildEnvironment.frompickle() is deprecated. ' + 'Please use pickle.load() instead.', + RemovedInSphinx30Warning) + with open(filename, 'rb') as f: + return cls.load(f, app) + + @staticmethod + def dump(env, f): + # type: (BuildEnvironment, IO) -> None + warnings.warn('BuildEnvironment.dump() is deprecated. ' + 'Please use pickle.dump() instead.', + RemovedInSphinx30Warning) + pickle.dump(env, f, pickle.HIGHEST_PROTOCOL) + + @classmethod + def dumps(cls, env): + # type: (BuildEnvironment) -> unicode + warnings.warn('BuildEnvironment.dumps() is deprecated. ' + 'Please use pickle.dumps() instead.', + RemovedInSphinx30Warning) + io = BytesIO() + cls.dump(env, io) + return io.getvalue() + + def topickle(self, filename): + # type: (unicode) -> None + warnings.warn('env.topickle() is deprecated. ' + 'Please use pickle.dump() instead.', + RemovedInSphinx30Warning) + with open(filename, 'wb') as f: + self.dump(self, f)