IntenseWebs/sphinx
3
0
Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2025-02-25 18:55:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4951 from tk0miya/refactor_env2

Browse Source 
Make Config and BuildEnvironment pickable
This commit is contained in:
Takeshi KOMIYA 2018-05-12 23:53:33 +09:00 committed by GitHub
commit 28b7b385c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 154 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGES
View File

@ -64,6 +64,12 @@ Deprecated
* ``sphinx.writers.latex.LaTeXWriter.restrict_footnote()`` is deprecated
* ``sphinx.writers.latex.LaTeXWriter.unrestrict_footnote()`` is deprecated
* ``LaTeXWriter.bibitems`` is deprecated
* ``BuildEnvironment.load()`` is deprecated
* ``BuildEnvironment.loads()`` is deprecated
* ``BuildEnvironment.frompickle()`` is deprecated
* ``BuildEnvironment.dump()`` is deprecated
* ``BuildEnvironment.dumps()`` is deprecated
* ``BuildEnvironment.topickle()`` is deprecated
For more details, see `deprecation APIs list
<http://www.sphinx-doc.org/en/master/extdev/index.html#deprecated-apis>`_

30
doc/extdev/index.rst
View File

@ -197,6 +197,36 @@ The following is a list of deprecated interface.
- 3.0
- :meth:`~sphinx.application.Sphinx.add_domain()`
* - ``BuildEnvironment.load()``
- 1.8
- 3.0
- ``pickle.load()``
* - ``BuildEnvironment.loads()``
- 1.8
- 3.0
- ``pickle.loads()``
* - ``BuildEnvironment.frompickle()``
- 1.8
- 3.0
- ``pickle.load()``
* - ``BuildEnvironment.dump()``
- 1.8
- 3.0
- ``pickle.dump()``
* - ``BuildEnvironment.dumps()``
- 1.8
- 3.0
- ``pickle.dumps()``
* - ``BuildEnvironment.topickle()``
- 1.8
- 3.0
- ``pickle.dump()``
* - ``BuildEnvironment._nitpick_ignore``
- 1.8
- 3.0

6
sphinx/application.py
View File

@ -22,6 +22,7 @@ from os import path
from docutils.parsers.rst import Directive, directives, roles
from six import itervalues
from six.moves import cPickle as pickle
from six.moves import cStringIO
import sphinx
@ -291,7 +292,10 @@ class Sphinx(object):
else:
try:
logger.info(bold(__('loading pickled environment... ')), nonl=True)
self.env = BuildEnvironment.frompickle(filename, self)
with open(filename, 'rb') as f:
self.env = pickle.load(f)
self.env.app = self
self.env.config.values = self.config.values
needed, reason = self.env.need_refresh(self)
if needed:
raise IOError(reason)

12
sphinx/builders/__init__.py
View File

@ -17,7 +17,6 @@ from docutils import nodes
from six.moves import cPickle as pickle
from sphinx.deprecation import RemovedInSphinx20Warning
from sphinx.environment import BuildEnvironment
from sphinx.environment.adapters.asset import ImageAdapter
from sphinx.errors import SphinxError
from sphinx.io import read_doc
@ -372,7 +371,8 @@ class Builder(object):
# save the environment
from sphinx.application import ENV_PICKLE_FILENAME
logger.info(bold(__('pickling environment... ')), nonl=True)
self.env.topickle(path.join(self.doctreedir, ENV_PICKLE_FILENAME))
with open(path.join(self.doctreedir, ENV_PICKLE_FILENAME), 'wb') as f:
pickle.dump(self.env, f, pickle.HIGHEST_PROTOCOL)
logger.info(__('done'))
# global actions
@ -492,16 +492,16 @@ class Builder(object):
self.env.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> unicode
# type: (List[unicode]) -> bytes
self.env.app = self.app
for docname in docs:
self.read_doc(docname)
# allow pickling self to send it back
return BuildEnvironment.dumps(self.env)
return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
def merge(docs, otherenv):
# type: (List[unicode], unicode) -> None
env = BuildEnvironment.loads(otherenv)
# type: (List[unicode], bytes) -> None
env = pickle.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
tasks = ParallelTasks(nproc)

34
sphinx/config.py
View File

@ -11,12 +11,15 @@
import re
import traceback
import types
import warnings
from collections import OrderedDict
from os import path, getenv
from typing import Any, NamedTuple, Union
from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types
from six import (
PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types, class_types
)
from sphinx.deprecation import RemovedInSphinx30Warning
from sphinx.errors import ConfigError, ExtensionError
@ -35,6 +38,7 @@ if False:
logger = logging.getLogger(__name__)
CONFIG_FILENAME = 'conf.py'
UNSERIALIZEABLE_TYPES = class_types + (types.ModuleType, types.FunctionType)
copyright_year_re = re.compile(r'^((\d{4}-)?)(\d{4})(?=[ ,])')
if PY3:
@ -308,6 +312,34 @@ class Config(object):
rebuild = [rebuild]
return (value for value in self if value.rebuild in rebuild)
def __getstate__(self):
# type: () -> Dict
"""Obtains serializable data for pickling."""
# remove potentially pickling-problematic values from config
__dict__ = {}
for key, value in iteritems(self.__dict__):
if key.startswith('_') or isinstance(value, UNSERIALIZEABLE_TYPES):
pass
else:
__dict__[key] = value
# create a picklable copy of values list
__dict__['values'] = {}
for key, value in iteritems(self.values): # type: ignore
real_value = getattr(self, key)
if isinstance(real_value, UNSERIALIZEABLE_TYPES):
# omit unserializable value
real_value = None
# types column is also omitted
__dict__['values'][key] = (real_value, value[1], None)
return __dict__
def __setstate__(self, state):
# type: (Dict) -> None
self.__dict__.update(state)
def eval_config_file(filename, tags):
# type: (unicode, Tags) -> Dict[unicode, Any]

140
sphinx/environment/__init__.py
View File

@ -12,14 +12,13 @@
import os
import re
import sys
import types
import warnings
from collections import defaultdict
from copy import copy
from os import path
from docutils.utils import get_source_line
from six import BytesIO, class_types, next
from six import BytesIO, next
from six.moves import cPickle as pickle
from sphinx import addnodes
@ -68,7 +67,7 @@ default_settings = {
# or changed to properly invalidate pickle files.
#
# NOTE: increase base version by 2 to have distinct numbers for Py2 and 3
ENV_VERSION = 52 + (sys.version_info[0] - 2)
ENV_VERSION = 53 + (sys.version_info[0] - 2)
versioning_conditions = {
@ -92,69 +91,6 @@ class BuildEnvironment(object):
domains = None # type: Dict[unicode, Domain]
# --------- ENVIRONMENT PERSISTENCE ----------------------------------------
@staticmethod
def load(f, app=None):
# type: (IO, Sphinx) -> BuildEnvironment
try:
env = pickle.load(f)
except Exception as exc:
# This can happen for example when the pickle is from a
# different version of Sphinx.
raise IOError(exc)
if app:
env.app = app
env.config.values = app.config.values
return env
@classmethod
def loads(cls, string, app=None):
# type: (unicode, Sphinx) -> BuildEnvironment
io = BytesIO(string)
return cls.load(io, app)
@classmethod
def frompickle(cls, filename, app):
# type: (unicode, Sphinx) -> BuildEnvironment
with open(filename, 'rb') as f:
return cls.load(f, app)
@staticmethod
def dump(env, f):
# type: (BuildEnvironment, IO) -> None
# remove unpicklable attributes
app = env.app
del env.app
values = env.config.values
del env.config.values
domains = env.domains
del env.domains
# remove potentially pickling-problematic values from config
for key, val in list(vars(env.config).items()):
if key.startswith('_') or \
isinstance(val, types.ModuleType) or \
isinstance(val, types.FunctionType) or \
isinstance(val, class_types):
del env.config[key]
pickle.dump(env, f, pickle.HIGHEST_PROTOCOL)
# reset attributes
env.domains = domains
env.config.values = values
env.app = app
@classmethod
def dumps(cls, env):
# type: (BuildEnvironment) -> unicode
io = BytesIO()
cls.dump(env, io)
return io.getvalue()
def topickle(self, filename):
# type: (unicode) -> None
with open(filename, 'wb') as f:
self.dump(self, f)
# --------- ENVIRONMENT INITIALIZATION -------------------------------------
def __init__(self, app):
@ -257,6 +193,17 @@ class BuildEnvironment(object):
# attributes of "any" cross references
self.ref_context = {} # type: Dict[unicode, Any]
def __getstate__(self):
# type: () -> Dict
"""Obtains serializable data for pickling."""
__dict__ = self.__dict__.copy()
__dict__.update(app=None, domains={}) # clear unpickable attributes
return __dict__
def __setstate__(self, state):
# type: (Dict) -> None
self.__dict__.update(state)
def set_warnfunc(self, func):
# type: (Callable) -> None
warnings.warn('env.set_warnfunc() is now deprecated. Use sphinx.util.logging instead.',
@ -824,3 +771,64 @@ class BuildEnvironment(object):
'Please use config.nitpick_ignore instead.',
RemovedInSphinx30Warning)
return self.config.nitpick_ignore
@staticmethod
def load(f, app=None):
# type: (IO, Sphinx) -> BuildEnvironment
warnings.warn('BuildEnvironment.load() is deprecated. '
'Please use pickle.load() instead.',
RemovedInSphinx30Warning)
try:
env = pickle.load(f)
except Exception as exc:
# This can happen for example when the pickle is from a
# different version of Sphinx.
raise IOError(exc)
if app:
env.app = app
env.config.values = app.config.values
return env
@classmethod
def loads(cls, string, app=None):
# type: (unicode, Sphinx) -> BuildEnvironment
warnings.warn('BuildEnvironment.loads() is deprecated. '
'Please use pickle.loads() instead.',
RemovedInSphinx30Warning)
io = BytesIO(string)
return cls.load(io, app)
@classmethod
def frompickle(cls, filename, app):
# type: (unicode, Sphinx) -> BuildEnvironment
warnings.warn('BuildEnvironment.frompickle() is deprecated. '
'Please use pickle.load() instead.',
RemovedInSphinx30Warning)
with open(filename, 'rb') as f:
return cls.load(f, app)
@staticmethod
def dump(env, f):
# type: (BuildEnvironment, IO) -> None
warnings.warn('BuildEnvironment.dump() is deprecated. '
'Please use pickle.dump() instead.',
RemovedInSphinx30Warning)
pickle.dump(env, f, pickle.HIGHEST_PROTOCOL)
@classmethod
def dumps(cls, env):
# type: (BuildEnvironment) -> unicode
warnings.warn('BuildEnvironment.dumps() is deprecated. '
'Please use pickle.dumps() instead.',
RemovedInSphinx30Warning)
io = BytesIO()
cls.dump(env, io)
return io.getvalue()
def topickle(self, filename):
# type: (unicode) -> None
warnings.warn('env.topickle() is deprecated. '
'Please use pickle.dump() instead.',
RemovedInSphinx30Warning)
with open(filename, 'wb') as f:
self.dump(self, f)