Merge pull request #3311 from tk0miya/split_app.status_iterator

Sphinx.status_iterator() is now deprecated

CHANGES

@@ -52,6 +52,8 @@ Deprecated
 * #3318: ``notice`` is now deprecated as LaTeX environment name and will be
   removed at Sphinx 1.7. Extension authors please use ``sphinxadmonition``
   instead (as Sphinx does since 1.5.)
+* ``Sphinx.status_iterator()` and ``Sphinx.old_status_iterator()`` is now
+  deprecated.  Please use ``sphinx.util:status_iterator()`` intead.
 
 Release 1.5.2 (in development)
 ===============================

sphinx/application.py

@@ -21,7 +21,7 @@ import traceback
 from os import path
 from collections import deque
 
-from six import iteritems, itervalues, text_type
+from six import iteritems, itervalues
 from six.moves import cStringIO
 
 from docutils import nodes
@@ -42,11 +42,10 @@ from sphinx.roles import XRefRole
 from sphinx.util import pycompat  # noqa: F401
 from sphinx.util import import_object
 from sphinx.util import logging
+from sphinx.util import status_iterator, old_status_iterator, display_chunk
 from sphinx.util.tags import Tags
 from sphinx.util.osutil import ENOENT
-from sphinx.util.console import (  # type: ignore
+from sphinx.util.console import bold, darkgreen  # type: ignore
-    bold, darkgreen, term_width_line
-)
 from sphinx.util.i18n import find_catalog_source_files
 
 if False:
@@ -431,48 +430,31 @@ class Sphinx(object):
 
     def _display_chunk(chunk):
         # type: (Any) -> unicode
-        if isinstance(chunk, (list, tuple)):
+        warnings.warn('app._display_chunk() is now deprecated. '
-            if len(chunk) == 1:
+                      'Use sphinx.util.display_chunk() instead.',
-                return text_type(chunk[0])
+                      RemovedInSphinx17Warning)
-            return '%s .. %s' % (chunk[0], chunk[-1])
+        return display_chunk(chunk)
-        return text_type(chunk)
 
     def old_status_iterator(self, iterable, summary, colorfunc=darkgreen,
-                            stringify_func=_display_chunk):
+                            stringify_func=display_chunk):
         # type: (Iterable, unicode, Callable, Callable[[Any], unicode]) -> Iterator
-        l = 0
+        warnings.warn('app.old_status_iterator() is now deprecated. '
-        for item in iterable:
+                      'Use sphinx.util.status_iterator() instead.',
-            if l == 0:
+                      RemovedInSphinx17Warning)
-                logger.info(bold(summary), nonl=True)
+        for item in old_status_iterator(iterable, summary,
-                l = 1
+                                        color="darkgreen", stringify_func=stringify_func):
-            logger.info(colorfunc(stringify_func(item)) + ' ', nonl=True)
             yield item
-        if l == 1:
-            logger.info('')
 
     # new version with progress info
     def status_iterator(self, iterable, summary, colorfunc=darkgreen, length=0,
                         stringify_func=_display_chunk):
         # type: (Iterable, unicode, Callable, int, Callable[[Any], unicode]) -> Iterable
-        if length == 0:
+        warnings.warn('app.status_iterator() is now deprecated. '
-            for item in self.old_status_iterator(iterable, summary, colorfunc,
+                      'Use sphinx.util.status_iterator() instead.',
-                                                 stringify_func):
+                      RemovedInSphinx17Warning)
-                yield item
+        for item in status_iterator(iterable, summary, length=length, verbosity=self.verbosity,
-            return
+                                    color="darkgreen", stringify_func=stringify_func):
-        l = 0
-        summary = bold(summary)
-        for item in iterable:
-            l += 1
-            s = '%s[%3d%%] %s' % (summary, 100*l/length,
-                                  colorfunc(stringify_func(item)))
-            if self.verbosity:
-                s += '\n'
-            else:
-                s = term_width_line(s)
-            logger.info(s, nonl=True)
             yield item
-        if l > 0:
-            logger.info('')
 
     # ---- general extensibility interface -------------------------------------
 

sphinx/builders/__init__.py

@@ -19,10 +19,10 @@ except ImportError:
 
 from docutils import nodes
 
-from sphinx.util import i18n, path_stabilize, logging
+from sphinx.util import i18n, path_stabilize, logging, status_iterator
 from sphinx.util.osutil import SEP, relative_uri
 from sphinx.util.i18n import find_catalog
-from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util.console import bold  # type: ignore
 from sphinx.util.parallel import ParallelTasks, SerialTasks, make_chunks, \
     parallel_available
 
@@ -183,9 +183,9 @@ class Builder(object):
             return path.relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP)
 
         logger.info(bold('building [mo]: ') + message)
-        for catalog in self.app.status_iterator(
+        for catalog in status_iterator(catalogs, 'writing output... ', "darkgreen",
-                catalogs, 'writing output... ', darkgreen, len(catalogs),
+                                       len(catalogs), self.app.verbosity,
-                cat2relpath):
+                                       stringify_func=cat2relpath):
             catalog.write_mo(self.config.language)
 
     def compile_all_catalogs(self):
@@ -384,8 +384,8 @@ class Builder(object):
     def _write_serial(self, docnames):
         # type: (Sequence[unicode]) -> None
         with logging.pending_warnings():
-            for docname in self.app.status_iterator(
+            for docname in status_iterator(docnames, 'writing output... ', "darkgreen",
-                    docnames, 'writing output... ', darkgreen, len(docnames)):
+                                           len(docnames), self.app.verbosity):
                 doctree = self.env.get_and_resolve_doctree(docname, self)
                 self.write_doc_serialized(docname, doctree)
                 self.write_doc(docname, doctree)
@@ -406,8 +406,8 @@ class Builder(object):
         tasks = ParallelTasks(nproc)
         chunks = make_chunks(docnames, nproc)
 
-        for chunk in self.app.status_iterator(
+        for chunk in status_iterator(chunks, 'writing output... ', "darkgreen",
-                chunks, 'writing output... ', darkgreen, len(chunks)):
+                                     len(chunks), self.app.verbosity):
             arg = []
             for i, docname in enumerate(chunk):
                 doctree = self.env.get_and_resolve_doctree(docname, self)

sphinx/builders/epub.py

@@ -30,9 +30,9 @@ from docutils import nodes
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.util import logging
+from sphinx.util import status_iterator
 from sphinx.util.osutil import ensuredir, copyfile, make_filename, EEXIST
 from sphinx.util.smartypants import sphinx_smarty_pants as ssp
-from sphinx.util.console import brown  # type: ignore
 
 if False:
     # For type annotation
@@ -470,8 +470,8 @@ class EpubBuilder(StandaloneHTMLBuilder):
         converting the format and resizing the image if necessary/possible.
         """
         ensuredir(path.join(self.outdir, self.imagedir))
-        for src in self.app.status_iterator(self.images, 'copying images... ',
+        for src in status_iterator(self.images, 'copying images... ', "brown",
-                                            brown, len(self.images)):
+                                   len(self.images), self.app.verbosity):
             dest = self.images[src]
             try:
                 img = Image.open(path.join(self.srcdir, src))

sphinx/builders/gettext.py

@@ -21,12 +21,12 @@ from uuid import uuid4
 from six import iteritems
 
 from sphinx.builders import Builder
-from sphinx.util import split_index_msg, logging
+from sphinx.util import split_index_msg, logging, status_iterator
 from sphinx.util.tags import Tags
 from sphinx.util.nodes import extract_messages, traverse_translatable_index
 from sphinx.util.osutil import safe_relpath, ensuredir, canon_path
 from sphinx.util.i18n import find_catalog
-from sphinx.util.console import darkgreen, purple, bold  # type: ignore
+from sphinx.util.console import bold  # type: ignore
 from sphinx.locale import pairindextypes
 
 if False:
@@ -224,8 +224,8 @@ class MessageCatalogBuilder(I18nBuilder):
 
         extract_translations = self.templates.environment.extract_translations
 
-        for template in self.app.status_iterator(
+        for template in status_iterator(files, 'reading templates... ', "purple",
-                files, 'reading templates... ', purple, len(files)):
+                                        len(files), self.app.verbosity):
             with open(template, 'r', encoding='utf-8') as f:  # type: ignore
                 context = f.read()
             for line, meth, msg in extract_translations(context):
@@ -247,10 +247,11 @@ class MessageCatalogBuilder(I18nBuilder):
             ctime = datetime.fromtimestamp(  # type: ignore
                 timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'),
         )
-        for textdomain, catalog in self.app.status_iterator(
+        for textdomain, catalog in status_iterator(iteritems(self.catalogs),
-                iteritems(self.catalogs), "writing message catalogs... ",
+                                                   "writing message catalogs... ",
-                darkgreen, len(self.catalogs),
+                                                   "darkgreen", len(self.catalogs),
-                lambda textdomain__: textdomain__[0]):
+                                                   self.app.verbosity,
+                                                   lambda textdomain__: textdomain__[0]):
             # noop if config.gettext_compact is set
             ensuredir(path.join(self.outdir, path.dirname(textdomain)))
 

sphinx/builders/html.py

@@ -28,7 +28,7 @@ from docutils.frontend import OptionParser
 from docutils.readers.doctree import Reader as DoctreeReader
 
 from sphinx import package_dir, __display_version__
-from sphinx.util import jsonimpl, logging
+from sphinx.util import jsonimpl, logging, status_iterator
 from sphinx.util.i18n import format_date
 from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \
     movefile, copyfile
@@ -42,7 +42,7 @@ from sphinx.theming import Theme
 from sphinx.builders import Builder
 from sphinx.application import ENV_PICKLE_FILENAME
 from sphinx.highlighting import PygmentsBridge
-from sphinx.util.console import bold, darkgreen, brown  # type: ignore
+from sphinx.util.console import bold, darkgreen  # type: ignore
 from sphinx.writers.html import HTMLWriter, HTMLTranslator, \
     SmartyPantsHTMLTranslator
 
@@ -584,8 +584,8 @@ class StandaloneHTMLBuilder(Builder):
         # copy image files
         if self.images:
             ensuredir(path.join(self.outdir, self.imagedir))
-            for src in self.app.status_iterator(self.images, 'copying images... ',
+            for src in status_iterator(self.images, 'copying images... ', "brown",
-                                                brown, len(self.images)):
+                                       len(self.images), self.app.verbosity):
                 dest = self.images[src]
                 try:
                     copyfile(path.join(self.srcdir, src),
@@ -601,10 +601,9 @@ class StandaloneHTMLBuilder(Builder):
         # copy downloadable files
         if self.env.dlfiles:
             ensuredir(path.join(self.outdir, '_downloads'))
-            for src in self.app.status_iterator(self.env.dlfiles,
+            for src in status_iterator(self.env.dlfiles, 'copying downloadable files... ',
-                                                'copying downloadable files... ',
+                                       "brown", len(self.env.dlfiles), self.app.verbosity,
-                                                brown, len(self.env.dlfiles),
+                                       stringify_func=to_relpath):
-                                                stringify_func=to_relpath):
                 dest = self.env.dlfiles[src][1]
                 try:
                     copyfile(path.join(self.srcdir, src),

sphinx/environment/__init__.py

@@ -33,14 +33,15 @@ from docutils.frontend import OptionParser
 
 from sphinx import addnodes
 from sphinx.io import SphinxStandaloneReader, SphinxDummyWriter, SphinxFileInput
-from sphinx.util import get_matching_docs, docname_join, FilenameUniqDict, logging
+from sphinx.util import logging
+from sphinx.util import get_matching_docs, docname_join, FilenameUniqDict, status_iterator
 from sphinx.util.nodes import clean_astext, WarningStream, is_translatable, \
     process_only_nodes
 from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir
 from sphinx.util.images import guess_mimetype
 from sphinx.util.i18n import find_catalog_files, get_image_filename_for_language, \
     search_image_for_language
-from sphinx.util.console import bold, purple  # type: ignore
+from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import sphinx_domains
 from sphinx.util.matching import compile_matchers
 from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks
@@ -623,8 +624,8 @@ class BuildEnvironment(object):
 
     def _read_serial(self, docnames, app):
         # type: (List[unicode], Sphinx) -> None
-        for docname in app.status_iterator(docnames, 'reading sources... ',
+        for docname in status_iterator(docnames, 'reading sources... ', "purple",
-                                           purple, len(docnames)):
+                                       len(docnames), self.app.verbosity):
             # remove all inventory entries for that file
             app.emit('env-purge-doc', self, docname)
             self.clear_doc(docname)
@@ -661,8 +662,8 @@ class BuildEnvironment(object):
         chunks = make_chunks(docnames, nproc)
 
         warnings = []  # type: List[Tuple]
-        for chunk in app.status_iterator(
+        for chunk in status_iterator(chunks, 'reading sources... ', "purple",
-                chunks, 'reading sources... ', purple, len(chunks)):
+                                     len(chunks), self.app.verbosity):
             tasks.add_task(read_process, chunk, merge)
 
         # make sure all threads have finished

sphinx/ext/viewcode.py

@@ -19,9 +19,8 @@ import sphinx
 from sphinx import addnodes
 from sphinx.locale import _
 from sphinx.pycode import ModuleAnalyzer
-from sphinx.util import get_full_modname, logging
+from sphinx.util import get_full_modname, logging, status_iterator
 from sphinx.util.nodes import make_refnode
-from sphinx.util.console import blue  # type: ignore
 
 if False:
     # For type annotation
@@ -147,9 +146,10 @@ def collect_pages(app):
 #    app.builder.info(' (%d module code pages)' %
 #                     len(env._viewcode_modules), nonl=1)
 
-    for modname, entry in app.status_iterator(
+    for modname, entry in status_iterator(iteritems(env._viewcode_modules),  # type: ignore
-            iteritems(env._viewcode_modules), 'highlighting module code... ',  # type:ignore
+                                          'highlighting module code... ', "blue",
-            blue, len(env._viewcode_modules), lambda x: x[0]):  # type:ignore
+                                          len(env._viewcode_modules),  # type: ignore
+                                          app.verbosity, lambda x: x[0]):
         if not entry:
             continue
         code, tags, used, refname = entry

sphinx/util/__init__.py

@@ -29,7 +29,7 @@ from docutils.utils import relative_path
 
 from sphinx.errors import PycodeError, SphinxParallelError, ExtensionError
 from sphinx.util import logging
-from sphinx.util.console import strip_colors
+from sphinx.util.console import strip_colors, colorize, bold, term_width_line  # type: ignore
 from sphinx.util.fileutil import copy_asset_file
 from sphinx.util.osutil import fs_encoding
 
@@ -45,7 +45,7 @@ from sphinx.util.matching import patfilter  # noqa
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Iterable, Pattern, Sequence, Tuple  # NOQA
+    from typing import Any, Callable, Iterable, Iterator, Pattern, Sequence, Tuple  # NOQA
 
 
 logger = logging.getLogger(__name__)
@@ -537,3 +537,49 @@ def split_docinfo(text):
         return '', result[0]
     else:
         return result[1:]
+
+
+def display_chunk(chunk):
+    # type: (Any) -> unicode
+    if isinstance(chunk, (list, tuple)):
+        if len(chunk) == 1:
+            return text_type(chunk[0])
+        return '%s .. %s' % (chunk[0], chunk[-1])
+    return text_type(chunk)
+
+
+def old_status_iterator(iterable, summary, color="darkgreen", stringify_func=display_chunk):
+    # type: (Iterable, unicode, str, Callable[[Any], unicode]) -> Iterator
+    l = 0
+    for item in iterable:
+        if l == 0:
+            logger.info(bold(summary), nonl=True)
+            l = 1
+        logger.info(stringify_func(item), color=color, nonl=True)
+        logger.info(" ", nonl=True)
+        yield item
+    if l == 1:
+        logger.info('')
+
+
+# new version with progress info
+def status_iterator(iterable, summary, color="darkgreen", length=0, verbosity=0,
+                    stringify_func=display_chunk):
+    # type: (Iterable, unicode, str, int, int, Callable[[Any], unicode]) -> Iterable
+    if length == 0:
+        for item in old_status_iterator(iterable, summary, color, stringify_func):
+            yield item
+        return
+    l = 0
+    summary = bold(summary)
+    for item in iterable:
+        l += 1
+        s = '%s[%3d%%] %s' % (summary, 100 * l / length, colorize(color, stringify_func(item)))
+        if verbosity:
+            s += '\n'
+        else:
+            s = term_width_line(s)
+        logger.info(s, nonl=True)
+        yield item
+    if l > 0:
+        logger.info('')

sphinx/util/console.py

@@ -83,7 +83,7 @@ def coloron():
 
 
 def colorize(name, text):
-    # type: (str, str) -> str
+    # type: (str, unicode) -> unicode
     return codes.get(name, '') + text + codes.get('reset', '')
 
 

tests/test_util.py

@@ -8,7 +8,14 @@
     :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-from sphinx.util import encode_uri, split_docinfo
+
+import pytest
+from mock import patch
+
+from sphinx.util import display_chunk, encode_uri, split_docinfo, status_iterator
+from sphinx.util import logging
+
+from util import with_app, strip_escseq
 
 
 def test_encode_uri():
@@ -46,3 +53,44 @@ def test_splitdocinfo():
     docinfo, content = split_docinfo(source)
     assert docinfo == ":multiline: one\n\ttwo\n\tthree\n"
     assert content == '\nHello world.\n'
+
+
+def test_display_chunk():
+    assert display_chunk('hello') == 'hello'
+    assert display_chunk(['hello']) == 'hello'
+    assert display_chunk(['hello', 'sphinx', 'world']) == 'hello .. world'
+    assert display_chunk(('hello',)) == 'hello'
+    assert display_chunk(('hello', 'sphinx', 'world')) == 'hello .. world'
+
+
+@pytest.mark.sphinx('dummy')
+@patch('sphinx.util.console._tw', 40)  # terminal width = 40
+def test_status_iterator(app, status, warning):
+    logging.setup(app, status, warning)
+
+    # test for old_status_iterator
+    status.truncate(0)
+    yields = list(status_iterator(['hello', 'sphinx', 'world'], 'testing ... '))
+    output = strip_escseq(status.getvalue())
+    assert 'testing ... hello sphinx world \n' in output
+    assert yields == ['hello', 'sphinx', 'world']
+
+    # test for status_iterator (verbosity=0)
+    status.truncate(0)
+    yields = list(status_iterator(['hello', 'sphinx', 'world'], 'testing ... ',
+                                  length=3, verbosity=0))
+    output = strip_escseq(status.getvalue())
+    assert 'testing ... [ 33%] hello                \r' in output
+    assert 'testing ... [ 66%] sphinx               \r' in output
+    assert 'testing ... [100%] world                \r\n' in output
+    assert yields == ['hello', 'sphinx', 'world']
+
+    # test for status_iterator (verbosity=1)
+    status.truncate(0)
+    yields = list(status_iterator(['hello', 'sphinx', 'world'], 'testing ... ',
+                                  length=3, verbosity=1))
+    output = strip_escseq(status.getvalue())
+    assert 'testing ... [ 33%] hello\n' in output
+    assert 'testing ... [ 66%] sphinx\n' in output
+    assert 'testing ... [100%] world\n\n' in output
+    assert yields == ['hello', 'sphinx', 'world']