From 10e231bf128c5f051fb48d97715b082fb2b9d4c7 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 25 Mar 2008 10:16:51 +0000 Subject: [PATCH] Add a dependency system for handling .. include, .. literalinclude and later .. image dependencies. --- CHANGES | 7 ++++ sphinx/builder.py | 36 ++++++++++---------- sphinx/directives.py | 5 +-- sphinx/environment.py | 78 +++++++++++++++++++++++++++++++------------ sphinx/ext/doctest.py | 4 +-- sphinx/linkcheck.py | 2 +- 6 files changed, 88 insertions(+), 44 deletions(-) diff --git a/CHANGES b/CHANGES index 4eb207fc3..ed0e56b2e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +Changes in trunk +================ + +* sphinx.environment: Take dependent files into account when collecting + the set of outdated sources. + + Release 0.1.61843 (Mar 24, 2008) ================================ diff --git a/sphinx/builder.py b/sphinx/builder.py index 00b8890d3..ca9456cd0 100644 --- a/sphinx/builder.py +++ b/sphinx/builder.py @@ -127,8 +127,7 @@ class Builder(object): # build methods def load_env(self): - """Set up the build environment. Return True if a pickled file could be - successfully loaded, False if a new environment had to be created.""" + """Set up the build environment.""" if self.env: return if not self.freshenv: @@ -143,8 +142,10 @@ class Builder(object): else: self.info('failed: %s' % err) self.env = BuildEnvironment(self.srcdir, self.doctreedir, self.config) + self.env.find_files(self.config) else: self.env = BuildEnvironment(self.srcdir, self.doctreedir, self.config) + self.env.find_files(self.config) self.env.set_warnfunc(self.warn) def build_all(self): @@ -171,10 +172,6 @@ class Builder(object): def build_update(self): """Only rebuild files changed or added since last build.""" to_build = self.get_outdated_docs() - if not to_build and self.env.all_docs: - # if there is nothing in all_docs, it's a fresh env - self.info(bold('no target files are out of date, exiting.')) - return if isinstance(to_build, str): self.build([], to_build) else: @@ -213,6 +210,10 @@ class Builder(object): # global actions self.info(bold('checking consistency...')) self.env.check_consistency() + else: + if not docnames: + self.info(bold('no targets are out of date.')) + return # another indirection to support methods which don't build files # individually @@ -222,14 +223,15 @@ class Builder(object): self.info(bold('finishing... ')) self.finish() if self.app._warncount: - self.info(bold('build succeeded, %s warnings.' % self.app._warncount)) + self.info(bold('build succeeded, %s warning%s.' % + (self.app._warncount, self.app._warncount != 1 and 's' or ''))) else: self.info(bold('build succeeded.')) def write(self, build_docnames, updated_docnames, method='update'): if build_docnames is None: # build_all - build_docnames = self.env.all_docs + build_docnames = self.env.found_docs if method == 'update': # build updated ones as well docnames = set(build_docnames) | set(updated_docnames) @@ -383,7 +385,7 @@ class StandaloneHTMLBuilder(Builder): self.handle_page(docname, ctx) def finish(self): - self.info(bold('writing additional files...')) + self.info(bold('writing additional files...'), nonl=1) # the global general index @@ -397,6 +399,7 @@ class StandaloneHTMLBuilder(Builder): genindexentries = self.env.index, genindexcounts = indexcounts, ) + self.info(' genindex', nonl=1) self.handle_page('genindex', genindexcontext, 'genindex.html') # the global module index @@ -442,21 +445,26 @@ class StandaloneHTMLBuilder(Builder): modindexentries = modindexentries, platforms = platforms, ) + self.info(' modindex', nonl=1) self.handle_page('modindex', modindexcontext, 'modindex.html') # the search page + self.info(' search', nonl=1) self.handle_page('search', {}, 'search.html') # additional pages from conf.py for pagename, template in self.config.html_additional_pages.items(): + self.info(' '+pagename, nonl=1) self.handle_page(pagename, {}, template) # the index page indextemplate = self.config.html_index if indextemplate: + self.info(' index', nonl=1) self.handle_page('index', {'indextemplate': indextemplate}, 'index.html') # copy static files + self.info() self.info(bold('copying static files...')) ensuredir(path.join(self.outdir, 'static')) staticdirnames = [path.join(path.dirname(__file__), 'static')] + \ @@ -481,10 +489,7 @@ class StandaloneHTMLBuilder(Builder): return docname + '.html' def get_outdated_docs(self): - for docname in get_matching_docs( - self.srcdir, self.config.source_suffix, - exclude=set(self.config.unused_docs), - prune=['_sources']): + for docname in self.env.found_docs: targetname = self.env.doc2path(docname, self.outdir, '.html') try: targetmtime = path.getmtime(targetname) @@ -566,10 +571,7 @@ class PickleHTMLBuilder(StandaloneHTMLBuilder): self.init_translator_class() def get_outdated_docs(self): - for docname in get_matching_docs( - self.srcdir, self.config.source_suffix, - exclude=set(self.config.unused_docs), - prune=['_sources']): + for docname in self.env.found_docs: targetname = self.env.doc2path(docname, self.outdir, '.fpickle') try: targetmtime = path.getmtime(targetname) diff --git a/sphinx/directives.py b/sphinx/directives.py index 95ac565ee..781ea2c22 100644 --- a/sphinx/directives.py +++ b/sphinx/directives.py @@ -664,10 +664,10 @@ def literalinclude_directive(name, arguments, options, content, lineno, if not state.document.settings.file_insertion_enabled: return [state.document.reporter.warning('File insertion disabled', line=lineno)] env = state.document.settings.env - fn = arguments[0] + rel_fn = arguments[0] source_dir = path.dirname(path.abspath(state_machine.input_lines.source( lineno - state_machine.input_offset - 1))) - fn = path.normpath(path.join(source_dir, fn)) + fn = path.normpath(path.join(source_dir, rel_fn)) try: f = open(fn) @@ -683,6 +683,7 @@ def literalinclude_directive(name, arguments, options, content, lineno, retnode['language'] = options['language'] if 'linenos' in options: retnode['linenos'] = True + state.document.settings.env.note_dependency(rel_fn) return [retnode] literalinclude_directive.options = {'linenos': directives.flag, diff --git a/sphinx/environment.py b/sphinx/environment.py index 34590ddc0..5f3d71c41 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -57,7 +57,7 @@ default_settings = { # This is increased every time a new environment attribute is added # to properly invalidate pickle files. -ENV_VERSION = 18 +ENV_VERSION = 19 def walk_depth(node, depth, maxdepth): @@ -218,8 +218,10 @@ class BuildEnvironment: # All "docnames" here are /-separated and relative and exclude the source suffix. self.found_docs = set() # contains all existing docnames - self.all_docs = {} # docname -> (mtime, md5sum) at the time of build + self.all_docs = {} # docname -> mtime at the time of build # contains all built docnames + self.dependencies = {} # docname -> set of dependent file names, relative to + # documentation root # File metadata self.metadata = {} # docname -> dict of metadata items @@ -278,6 +280,7 @@ class BuildEnvironment: if docname in self.all_docs: self.all_docs.pop(docname, None) self.metadata.pop(docname, None) + self.dependencies.pop(docname, None) self.titles.pop(docname, None) self.tocs.pop(docname, None) self.toc_num_entries.pop(docname, None) @@ -318,14 +321,18 @@ class BuildEnvironment: else: return path.join(base, docname.replace(SEP, path.sep)) + suffix - def get_outdated_files(self, config, config_changed): + def find_files(self, config): """ - Return (added, changed, removed) sets. + Find all source files in the source dir and put them in self.found_docs. """ self.found_docs = set(get_matching_docs(self.srcdir, config.source_suffix, exclude=set(config.unused_docs), prune=['_sources'])) + def get_outdated_files(self, config_changed): + """ + Return (added, changed, removed) sets. + """ # clear all files no longer present removed = set(self.all_docs) - self.found_docs @@ -339,17 +346,28 @@ class BuildEnvironment: for docname in self.found_docs: if docname not in self.all_docs: added.add(docname) - else: - # if the doctree file is not there, rebuild - if not path.isfile(self.doc2path(docname, self.doctreedir, - '.doctree')): - changed.add(docname) - continue - mtime, md5sum = self.all_docs[docname] - newmtime = path.getmtime(self.doc2path(docname)) - if newmtime == mtime: - continue + continue + # if the doctree file is not there, rebuild + if not path.isfile(self.doc2path(docname, self.doctreedir, + '.doctree')): changed.add(docname) + continue + # check the mtime of the document + mtime = self.all_docs[docname] + newmtime = path.getmtime(self.doc2path(docname)) + if newmtime > mtime: + changed.add(docname) + continue + # finally, check the mtime of dependencies + for dep in self.dependencies.get(docname, ()): + deppath = path.join(self.srcdir, dep) + if not path.isfile(deppath): + changed.add(docname) + break + depmtime = path.getmtime(deppath) + if depmtime > mtime: + changed.add(docname) + break return added, changed, removed @@ -369,12 +387,14 @@ class BuildEnvironment: continue if not hasattr(self.config, key) or \ self.config[key] != config[key]: + msg = '[config changed] ' config_changed = True break else: msg = '' - added, changed, removed = self.get_outdated_files(config, config_changed) + self.find_files(config) + added, changed, removed = self.get_outdated_files(config_changed) msg += '%s added, %s changed, %s removed' % (len(added), len(changed), len(removed)) yield msg @@ -409,18 +429,14 @@ class BuildEnvironment: doctree = publish_doctree(None, src_path, FileInput, settings_overrides=self.settings, reader=MyStandaloneReader()) + self.process_dependencies(docname, doctree) self.process_metadata(docname, doctree) self.create_title_from(docname, doctree) self.note_labels_from(docname, doctree) self.build_toc_from(docname, doctree) - # calculate the MD5 of the file at time of build - f = open(src_path, 'rb') - try: - md5sum = md5(f.read()).digest() - finally: - f.close() - self.all_docs[docname] = (path.getmtime(src_path), md5sum) + # store time of reading, used to find outdated files + self.all_docs[docname] = time.time() if app: app.emit('doctree-read', doctree) @@ -430,6 +446,7 @@ class BuildEnvironment: doctree.transformer = None doctree.settings.warning_stream = None doctree.settings.env = None + doctree.settings.record_dependencies = None # cleanup self.docname = None @@ -452,6 +469,18 @@ class BuildEnvironment: else: return doctree + def process_dependencies(self, docname, doctree): + """ + Process docutils-generated dependency info. + """ + deps = doctree.settings.record_dependencies + if not deps: + return + basename = path.dirname(self.doc2path(docname, base=None)) + for dep in deps.list: + dep = path.join(basename, dep) + self.dependencies.setdefault(docname, set()).add(dep) + def process_metadata(self, docname, doctree): """ Process the docinfo part of the doctree as metadata. @@ -602,6 +631,11 @@ class BuildEnvironment: def note_versionchange(self, type, version, node, lineno): self.versionchanges.setdefault(version, []).append( (type, self.docname, lineno, self.currmodule, self.currdesc, node.astext())) + + def note_dependency(self, filename): + basename = path.dirname(self.doc2path(self.docname, base=None)) + filename = path.join(basename, filename) + self.dependencies.setdefault(self.docname, set()).add(filename) # ------- # --------- RESOLVING REFERENCES AND TOCTREES ------------------------------ diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py index 64b7856cf..ae69dd974 100644 --- a/sphinx/ext/doctest.py +++ b/sphinx/ext/doctest.py @@ -183,7 +183,7 @@ Results of doctest builder run on %s return '' def get_outdated_docs(self): - return self.env.all_docs + return self.env.found_docs def finish(self): # write executive summary @@ -204,7 +204,7 @@ Doctest summary def write(self, build_docnames, updated_docnames, method='update'): if build_docnames is None: - build_docnames = self.env.all_docs + build_docnames = sorted(self.env.all_docs) self.info(bold('running tests...')) for docname in build_docnames: diff --git a/sphinx/linkcheck.py b/sphinx/linkcheck.py index f67e11948..d7877a8c7 100644 --- a/sphinx/linkcheck.py +++ b/sphinx/linkcheck.py @@ -42,7 +42,7 @@ class CheckExternalLinksBuilder(Builder): return '' def get_outdated_docs(self): - return self.env.all_docs + return self.env.found_docs def prepare_writing(self, docnames): return