Add a dependency system for handling .. include, .. literalinclude

and later .. image dependencies.

CHANGES

@@ -1,3 +1,10 @@
+Changes in trunk
+================
+
+* sphinx.environment: Take dependent files into account when collecting
+  the set of outdated sources.
+
+
 Release 0.1.61843 (Mar 24, 2008)
 ================================
 

sphinx/builder.py

@@ -127,8 +127,7 @@ class Builder(object):
     # build methods
 
     def load_env(self):
-        """Set up the build environment. Return True if a pickled file could be
+        """Set up the build environment."""
-           successfully loaded, False if a new environment had to be created."""
         if self.env:
             return
         if not self.freshenv:
@@ -143,8 +142,10 @@ class Builder(object):
                 else:
                     self.info('failed: %s' % err)
                 self.env = BuildEnvironment(self.srcdir, self.doctreedir, self.config)
+                self.env.find_files(self.config)
         else:
             self.env = BuildEnvironment(self.srcdir, self.doctreedir, self.config)
+            self.env.find_files(self.config)
         self.env.set_warnfunc(self.warn)
 
     def build_all(self):
@@ -171,10 +172,6 @@ class Builder(object):
     def build_update(self):
         """Only rebuild files changed or added since last build."""
         to_build = self.get_outdated_docs()
-        if not to_build and self.env.all_docs:
-            # if there is nothing in all_docs, it's a fresh env
-            self.info(bold('no target files are out of date, exiting.'))
-            return
         if isinstance(to_build, str):
             self.build([], to_build)
         else:
@@ -213,6 +210,10 @@ class Builder(object):
             # global actions
             self.info(bold('checking consistency...'))
             self.env.check_consistency()
+        else:
+            if not docnames:
+                self.info(bold('no targets are out of date.'))
+                return
 
         # another indirection to support methods which don't build files
         # individually
@@ -222,14 +223,15 @@ class Builder(object):
         self.info(bold('finishing... '))
         self.finish()
         if self.app._warncount:
-            self.info(bold('build succeeded, %s warnings.' % self.app._warncount))
+            self.info(bold('build succeeded, %s warning%s.' %
+                           (self.app._warncount, self.app._warncount != 1 and 's' or '')))
         else:
             self.info(bold('build succeeded.'))
 
     def write(self, build_docnames, updated_docnames, method='update'):
         if build_docnames is None:
             # build_all
-            build_docnames = self.env.all_docs
+            build_docnames = self.env.found_docs
         if method == 'update':
             # build updated ones as well
             docnames = set(build_docnames) | set(updated_docnames)
@@ -383,7 +385,7 @@ class StandaloneHTMLBuilder(Builder):
         self.handle_page(docname, ctx)
 
     def finish(self):
-        self.info(bold('writing additional files...'))
+        self.info(bold('writing additional files...'), nonl=1)
 
         # the global general index
 
@@ -397,6 +399,7 @@ class StandaloneHTMLBuilder(Builder):
             genindexentries = self.env.index,
             genindexcounts = indexcounts,
         )
+        self.info(' genindex', nonl=1)
         self.handle_page('genindex', genindexcontext, 'genindex.html')
 
         # the global module index
@@ -442,21 +445,26 @@ class StandaloneHTMLBuilder(Builder):
                 modindexentries = modindexentries,
                 platforms = platforms,
             )
+            self.info(' modindex', nonl=1)
             self.handle_page('modindex', modindexcontext, 'modindex.html')
 
         # the search page
+        self.info(' search', nonl=1)
         self.handle_page('search', {}, 'search.html')
 
         # additional pages from conf.py
         for pagename, template in self.config.html_additional_pages.items():
+            self.info(' '+pagename, nonl=1)
             self.handle_page(pagename, {}, template)
 
         # the index page
         indextemplate = self.config.html_index
         if indextemplate:
+            self.info(' index', nonl=1)
             self.handle_page('index', {'indextemplate': indextemplate}, 'index.html')
 
         # copy static files
+        self.info()
         self.info(bold('copying static files...'))
         ensuredir(path.join(self.outdir, 'static'))
         staticdirnames = [path.join(path.dirname(__file__), 'static')] + \
@@ -481,10 +489,7 @@ class StandaloneHTMLBuilder(Builder):
         return docname + '.html'
 
     def get_outdated_docs(self):
-        for docname in get_matching_docs(
+        for docname in self.env.found_docs:
-            self.srcdir, self.config.source_suffix,
-            exclude=set(self.config.unused_docs),
-            prune=['_sources']):
             targetname = self.env.doc2path(docname, self.outdir, '.html')
             try:
                 targetmtime = path.getmtime(targetname)
@@ -566,10 +571,7 @@ class PickleHTMLBuilder(StandaloneHTMLBuilder):
         self.init_translator_class()
 
     def get_outdated_docs(self):
-        for docname in get_matching_docs(
+        for docname in self.env.found_docs:
-            self.srcdir, self.config.source_suffix,
-            exclude=set(self.config.unused_docs),
-            prune=['_sources']):
             targetname = self.env.doc2path(docname, self.outdir, '.fpickle')
             try:
                 targetmtime = path.getmtime(targetname)

sphinx/directives.py

@@ -664,10 +664,10 @@ def literalinclude_directive(name, arguments, options, content, lineno,
     if not state.document.settings.file_insertion_enabled:
         return [state.document.reporter.warning('File insertion disabled', line=lineno)]
     env = state.document.settings.env
-    fn = arguments[0]
+    rel_fn = arguments[0]
     source_dir = path.dirname(path.abspath(state_machine.input_lines.source(
         lineno - state_machine.input_offset - 1)))
-    fn = path.normpath(path.join(source_dir, fn))
+    fn = path.normpath(path.join(source_dir, rel_fn))
 
     try:
         f = open(fn)
@@ -683,6 +683,7 @@ def literalinclude_directive(name, arguments, options, content, lineno,
             retnode['language'] = options['language']
         if 'linenos' in options:
             retnode['linenos'] = True
+        state.document.settings.env.note_dependency(rel_fn)
     return [retnode]
 
 literalinclude_directive.options = {'linenos': directives.flag,

sphinx/environment.py

@@ -57,7 +57,7 @@ default_settings = {
 
 # This is increased every time a new environment attribute is added
 # to properly invalidate pickle files.
-ENV_VERSION = 18
+ENV_VERSION = 19
 
 
 def walk_depth(node, depth, maxdepth):
@@ -218,8 +218,10 @@ class BuildEnvironment:
         # All "docnames" here are /-separated and relative and exclude the source suffix.
 
         self.found_docs = set()     # contains all existing docnames
-        self.all_docs = {}          # docname -> (mtime, md5sum) at the time of build
+        self.all_docs = {}          # docname -> mtime at the time of build
                                     # contains all built docnames
+        self.dependencies = {}      # docname -> set of dependent file names, relative to
+                                    # documentation root
 
         # File metadata
         self.metadata = {}          # docname -> dict of metadata items
@@ -278,6 +280,7 @@ class BuildEnvironment:
         if docname in self.all_docs:
             self.all_docs.pop(docname, None)
             self.metadata.pop(docname, None)
+            self.dependencies.pop(docname, None)
             self.titles.pop(docname, None)
             self.tocs.pop(docname, None)
             self.toc_num_entries.pop(docname, None)
@@ -318,14 +321,18 @@ class BuildEnvironment:
         else:
             return path.join(base, docname.replace(SEP, path.sep)) + suffix
 
-    def get_outdated_files(self, config, config_changed):
+    def find_files(self, config):
         """
-        Return (added, changed, removed) sets.
+        Find all source files in the source dir and put them in self.found_docs.
         """
         self.found_docs = set(get_matching_docs(self.srcdir, config.source_suffix,
                                                 exclude=set(config.unused_docs),
                                                 prune=['_sources']))
 
+    def get_outdated_files(self, config_changed):
+        """
+        Return (added, changed, removed) sets.
+        """
         # clear all files no longer present
         removed = set(self.all_docs) - self.found_docs
 
@@ -339,17 +346,28 @@ class BuildEnvironment:
             for docname in self.found_docs:
                 if docname not in self.all_docs:
                     added.add(docname)
-                else:
+                    continue
-                    # if the doctree file is not there, rebuild
+                # if the doctree file is not there, rebuild
-                    if not path.isfile(self.doc2path(docname, self.doctreedir,
+                if not path.isfile(self.doc2path(docname, self.doctreedir,
-                                                     '.doctree')):
+                                                 '.doctree')):
-                        changed.add(docname)
-                        continue
-                    mtime, md5sum = self.all_docs[docname]
-                    newmtime = path.getmtime(self.doc2path(docname))
-                    if newmtime == mtime:
-                        continue
                     changed.add(docname)
+                    continue
+                # check the mtime of the document
+                mtime = self.all_docs[docname]
+                newmtime = path.getmtime(self.doc2path(docname))
+                if newmtime > mtime:
+                    changed.add(docname)
+                    continue
+                # finally, check the mtime of dependencies
+                for dep in self.dependencies.get(docname, ()):
+                    deppath = path.join(self.srcdir, dep)
+                    if not path.isfile(deppath):
+                        changed.add(docname)
+                        break
+                    depmtime = path.getmtime(deppath)
+                    if depmtime > mtime:
+                        changed.add(docname)
+                        break
 
         return added, changed, removed
 
@@ -369,12 +387,14 @@ class BuildEnvironment:
                     continue
                 if not hasattr(self.config, key) or \
                    self.config[key] != config[key]:
+
                     msg = '[config changed] '
                     config_changed = True
                     break
             else:
                 msg = ''
-        added, changed, removed = self.get_outdated_files(config, config_changed)
+        self.find_files(config)
+        added, changed, removed = self.get_outdated_files(config_changed)
         msg += '%s added, %s changed, %s removed' % (len(added), len(changed),
                                                      len(removed))
         yield msg
@@ -409,18 +429,14 @@ class BuildEnvironment:
         doctree = publish_doctree(None, src_path, FileInput,
                                   settings_overrides=self.settings,
                                   reader=MyStandaloneReader())
+        self.process_dependencies(docname, doctree)
         self.process_metadata(docname, doctree)
         self.create_title_from(docname, doctree)
         self.note_labels_from(docname, doctree)
         self.build_toc_from(docname, doctree)
 
-        # calculate the MD5 of the file at time of build
+        # store time of reading, used to find outdated files
-        f = open(src_path, 'rb')
+        self.all_docs[docname] = time.time()
-        try:
-            md5sum = md5(f.read()).digest()
-        finally:
-            f.close()
-        self.all_docs[docname] = (path.getmtime(src_path), md5sum)
 
         if app:
             app.emit('doctree-read', doctree)
@@ -430,6 +446,7 @@ class BuildEnvironment:
         doctree.transformer = None
         doctree.settings.warning_stream = None
         doctree.settings.env = None
+        doctree.settings.record_dependencies = None
 
         # cleanup
         self.docname = None
@@ -452,6 +469,18 @@ class BuildEnvironment:
         else:
             return doctree
 
+    def process_dependencies(self, docname, doctree):
+        """
+        Process docutils-generated dependency info.
+        """
+        deps = doctree.settings.record_dependencies
+        if not deps:
+            return
+        basename = path.dirname(self.doc2path(docname, base=None))
+        for dep in deps.list:
+            dep = path.join(basename, dep)
+            self.dependencies.setdefault(docname, set()).add(dep)
+
     def process_metadata(self, docname, doctree):
         """
         Process the docinfo part of the doctree as metadata.
@@ -602,6 +631,11 @@ class BuildEnvironment:
     def note_versionchange(self, type, version, node, lineno):
         self.versionchanges.setdefault(version, []).append(
             (type, self.docname, lineno, self.currmodule, self.currdesc, node.astext()))
+
+    def note_dependency(self, filename):
+        basename = path.dirname(self.doc2path(self.docname, base=None))
+        filename = path.join(basename, filename)
+        self.dependencies.setdefault(self.docname, set()).add(filename)
     # -------
 
     # --------- RESOLVING REFERENCES AND TOCTREES ------------------------------

sphinx/ext/doctest.py

@@ -183,7 +183,7 @@ Results of doctest builder run on %s
         return ''
 
     def get_outdated_docs(self):
-        return self.env.all_docs
+        return self.env.found_docs
 
     def finish(self):
         # write executive summary
@@ -204,7 +204,7 @@ Doctest summary
 
     def write(self, build_docnames, updated_docnames, method='update'):
         if build_docnames is None:
-            build_docnames = self.env.all_docs
+            build_docnames = sorted(self.env.all_docs)
 
         self.info(bold('running tests...'))
         for docname in build_docnames:

sphinx/linkcheck.py

@@ -42,7 +42,7 @@ class CheckExternalLinksBuilder(Builder):
         return ''
 
     def get_outdated_docs(self):
-        return self.env.all_docs
+        return self.env.found_docs
 
     def prepare_writing(self, docnames):
         return