Merge pull request #3416 from tk0miya/refactor_literalinclude

Refactor literalinclude

CHANGES

@@ -16,6 +16,8 @@ Incompatible changes
 * option directive also allows all punctuations for the option name (refs: #3366)
 * #3413: if :rst:dir:`literalinclude`'s ``:start-after:`` is used, make ``:lines:``
   relative (refs #3412)
+* ``literalinclude`` directive does not allow the combination of ``:diff:``
+  option and other options (refs: #3416)
 
 Features removed
 ----------------
@@ -58,10 +60,14 @@ Features added
   (but does not implement text flow around table).
 * latex: footnotes from inside tables are hyperlinked (except from captions or
   headers) (refs: #3422)
+* Emit warning if over dedent has detected on ``literalinclude`` directive
+  (refs: #3416)
 
 Bugs fixed
 ----------
 
+* ``literalinclude`` directive expands tabs after dedent-ing (refs: #3416)
+
 Deprecated
 ----------
 
@@ -103,6 +109,8 @@ Features added
 
 * Support requests-2.0.0 (experimental) (refs: #3367)
 * (latex) PDF page margin dimensions may be customized (refs: #3387)
+* ``literalinclude`` directive allows combination of ``:pyobject:`` and
+  ``:lines:`` options (refs: #3416)
 
 Bugs fixed
 ----------

Makefile

@@ -3,15 +3,12 @@ PYTHON ?= python
 .PHONY: all style-check type-check clean clean-pyc clean-patchfiles clean-backupfiles \
         clean-generated pylint reindent test covertest build
 
-DONT_CHECK = -i build -i dist -i sphinx/style/jquery.js \
-             -i sphinx/pycode/pgen2 -i sphinx/util/smartypants.py \
-             -i .ropeproject -i doc/_build -i tests/path.py \
-             -i utils/convert.py \
-             -i tests/typing_test_data.py \
-             -i tests/test_autodoc_py35.py \
-             -i tests/roots/test-warnings/undecodable.rst \
-             -i tests/build \
-             -i tests/roots/test-warnings/undecodable.rst \
+DONT_CHECK = -i .ropeproject \
+             -i .tox \
+             -i build \
+             -i dist \
+             -i doc/_build \
+             -i sphinx/pycode/pgen2 \
              -i sphinx/search/da.py \
              -i sphinx/search/de.py \
              -i sphinx/search/en.py \
@@ -28,7 +25,15 @@ DONT_CHECK = -i build -i dist -i sphinx/style/jquery.js \
              -i sphinx/search/ru.py \
              -i sphinx/search/sv.py \
              -i sphinx/search/tr.py \
-             -i .tox
+             -i sphinx/style/jquery.js \
+             -i sphinx/util/smartypants.py \
+             -i tests/build \
+             -i tests/path.py \
+             -i tests/roots/test-directive-code/target.py \
+             -i tests/roots/test-warnings/undecodable.rst \
+             -i tests/test_autodoc_py35.py \
+             -i tests/typing_test_data.py \
+             -i utils/convert.py
 
 all: clean-pyc clean-backupfiles style-check type-check test
 

sphinx/directives/code.py

@@ -11,21 +11,23 @@ import sys
 import codecs
 from difflib import unified_diff
 
-from six import string_types
-
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
 from docutils.statemachine import ViewList
 
 from sphinx import addnodes
 from sphinx.locale import _
+from sphinx.util import logging
 from sphinx.util import parselinenos
 from sphinx.util.nodes import set_source_info
 
 if False:
     # For type annotation
-    from typing import Any  # NOQA
+    from typing import Any, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
+    from sphinx.config import Config  # NOQA
+
+logger = logging.getLogger(__name__)
 
 
 class Highlight(Directive):
@@ -55,11 +57,14 @@ class Highlight(Directive):
                                        linenothreshold=linenothreshold)]
 
 
-def dedent_lines(lines, dedent):
-    # type: (List[unicode], int) -> List[unicode]
+def dedent_lines(lines, dedent, location=None):
+    # type: (List[unicode], int, Any) -> List[unicode]
     if not dedent:
         return lines
 
+    if any(s[:dedent].strip() for s in lines):
+        logger.warning(_('Over dedent has detected'), location=location)
+
     new_lines = []
     for line in lines:
         new_line = line[dedent:]
@@ -78,7 +83,8 @@ def container_wrapper(directive, literal_node, caption):
     directive.state.nested_parse(ViewList([caption], source=''),
                                  directive.content_offset, parsed)
     if isinstance(parsed[0], nodes.system_message):
-        raise ValueError(parsed[0])
+        msg = _('Invalid caption: %s' % parsed[0].astext())
+        raise ValueError(msg)
     caption_node = nodes.caption(parsed[0].rawsource, '',
                                  *parsed[0].children)
     caption_node.source = literal_node.source
@@ -110,22 +116,30 @@ class CodeBlock(Directive):
 
     def run(self):
         # type: () -> List[nodes.Node]
+        document = self.state.document
         code = u'\n'.join(self.content)
+        location = self.state_machine.get_source_and_line(self.lineno)
 
         linespec = self.options.get('emphasize-lines')
         if linespec:
             try:
                 nlines = len(self.content)
-                hl_lines = [x + 1 for x in parselinenos(linespec, nlines)]
+                hl_lines = parselinenos(linespec, nlines)
+                if any(i >= nlines for i in hl_lines):
+                    logger.warning('line number spec is out of range(1-%d): %r' %
+                                   (nlines, self.options['emphasize_lines']),
+                                   location=location)
+
+                hl_lines = [x + 1 for x in hl_lines if x < nlines]
             except ValueError as err:
-                document = self.state.document
                 return [document.reporter.warning(str(err), line=self.lineno)]
         else:
             hl_lines = None
 
         if 'dedent' in self.options:
+            location = self.state_machine.get_source_and_line(self.lineno)
             lines = code.split('\n')
-            lines = dedent_lines(lines, self.options['dedent'])
+            lines = dedent_lines(lines, self.options['dedent'], location=location)
             code = '\n'.join(lines)
 
         literal = nodes.literal_block(code, code)
@@ -145,9 +159,7 @@ class CodeBlock(Directive):
             try:
                 literal = container_wrapper(self, literal, caption)
             except ValueError as exc:
-                document = self.state.document
-                errmsg = _('Invalid caption: %s' % exc[0][0].astext())  # type: ignore
-                return [document.reporter.warning(errmsg, line=self.lineno)]
+                return [document.reporter.warning(str(exc), line=self.lineno)]
 
         # literal will be note_implicit_target that is linked from caption and numref.
         # when options['name'] is provided, it should be primary ID.
@@ -156,6 +168,196 @@ class CodeBlock(Directive):
         return [literal]
 
 
+class LiteralIncludeReader(object):
+    INVALID_OPTIONS_PAIR = [
+        ('lineno-match', 'lineno-start'),
+        ('lineno-match', 'append'),
+        ('lineno-match', 'prepend'),
+        ('start-after', 'start-at'),
+        ('end-before', 'end-at'),
+        ('diff', 'pyobject'),
+        ('diff', 'lineno-start'),
+        ('diff', 'lineno-match'),
+        ('diff', 'lines'),
+        ('diff', 'start-after'),
+        ('diff', 'end-before'),
+        ('diff', 'start-at'),
+        ('diff', 'end-at'),
+    ]
+
+    def __init__(self, filename, options, config):
+        # type: (unicode, Dict, Config) -> None
+        self.filename = filename
+        self.options = options
+        self.encoding = options.get('encoding', config.source_encoding)
+        self.lineno_start = self.options.get('lineno-start', 1)
+
+        self.parse_options()
+
+    def parse_options(self):
+        # type: () -> None
+        for option1, option2 in self.INVALID_OPTIONS_PAIR:
+            if option1 in self.options and option2 in self.options:
+                raise ValueError(_('Cannot use both "%s" and "%s" options') %
+                                 (option1, option2))
+
+    def read_file(self, filename, location=None):
+        # type: (unicode, Any) -> List[unicode]
+        try:
+            with codecs.open(filename, 'r', self.encoding, errors='strict') as f:  # type: ignore  # NOQA
+                text = f.read()  # type: unicode
+                if 'tab-width' in self.options:
+                    text = text.expandtabs(self.options['tab-width'])
+
+                lines = text.splitlines(True)
+                if 'dedent' in self.options:
+                    return dedent_lines(lines, self.options.get('dedent'), location=location)
+                else:
+                    return lines
+        except (IOError, OSError):
+            raise IOError(_('Include file %r not found or reading it failed') % filename)
+        except UnicodeError:
+            raise UnicodeError(_('Encoding %r used for reading included file %r seems to '
+                                 'be wrong, try giving an :encoding: option') %
+                               (self.encoding, filename))
+
+    def read(self, location=None):
+        # type: (Any) -> Tuple[unicode, int]
+        if 'diff' in self.options:
+            lines = self.show_diff()
+        else:
+            filters = [self.pyobject_filter,
+                       self.start_filter,
+                       self.end_filter,
+                       self.lines_filter,
+                       self.prepend_filter,
+                       self.append_filter]
+            lines = self.read_file(self.filename, location=location)
+            for func in filters:
+                lines = func(lines, location=location)
+
+        return ''.join(lines), len(lines)
+
+    def show_diff(self, location=None):
+        # type: (Any) -> List[unicode]
+        new_lines = self.read_file(self.filename)
+        old_filename = self.options.get('diff')
+        old_lines = self.read_file(old_filename)
+        diff = unified_diff(old_lines, new_lines, old_filename, self.filename)  # type: ignore
+        return list(diff)
+
+    def pyobject_filter(self, lines, location=None):
+        # type: (List[unicode], Any) -> List[unicode]
+        pyobject = self.options.get('pyobject')
+        if pyobject:
+            from sphinx.pycode import ModuleAnalyzer
+            analyzer = ModuleAnalyzer.for_file(self.filename, '')
+            tags = analyzer.find_tags()
+            if pyobject not in tags:
+                raise ValueError(_('Object named %r not found in include file %r') %
+                                 (pyobject, self.filename))
+            else:
+                start = tags[pyobject][1]
+                end = tags[pyobject][2]
+                lines = lines[start - 1:end - 1]
+                if 'lineno-match' in self.options:
+                    self.lineno_start = start
+
+        return lines
+
+    def lines_filter(self, lines, location=None):
+        # type: (List[unicode], Any) -> List[unicode]
+        linespec = self.options.get('lines')
+        if linespec:
+            linelist = parselinenos(linespec, len(lines))
+            if any(i >= len(lines) for i in linelist):
+                logger.warning('line number spec is out of range(1-%d): %r' %
+                               (len(lines), linespec), location=location)
+
+            if 'lineno-match' in self.options:
+                # make sure the line list is not "disjoint".
+                first = linelist[0]
+                if all(first + i == n for i, n in enumerate(linelist)):
+                    self.lineno_start += linelist[0]
+                else:
+                    raise ValueError(_('Cannot use "lineno-match" with a disjoint '
+                                       'set of "lines"'))
+
+            lines = [lines[n] for n in linelist if n < len(lines)]
+            if lines == []:
+                raise ValueError(_('Line spec %r: no lines pulled from include file %r') %
+                                 (linespec, self.filename))
+
+        return lines
+
+    def start_filter(self, lines, location=None):
+        # type: (List[unicode], Any) -> List[unicode]
+        if 'start-at' in self.options:
+            start = self.options.get('start-at')
+            inclusive = False
+        elif 'start-after' in self.options:
+            start = self.options.get('start-after')
+            inclusive = True
+        else:
+            start = None
+
+        if start:
+            for lineno, line in enumerate(lines):
+                if start in line:
+                    if inclusive:
+                        if 'lineno-match' in self.options:
+                            self.lineno_start += lineno + 1
+
+                        return lines[lineno + 1:]
+                    else:
+                        if 'lineno-match' in self.options:
+                            self.lineno_start += lineno
+
+                        return lines[lineno:]
+
+        return lines
+
+    def end_filter(self, lines, location=None):
+        # type: (List[unicode], Any) -> List[unicode]
+        if 'end-at' in self.options:
+            end = self.options.get('end-at')
+            inclusive = True
+        elif 'end-before' in self.options:
+            end = self.options.get('end-before')
+            inclusive = False
+        else:
+            end = None
+
+        if end:
+            for lineno, line in enumerate(lines):
+                if end in line:
+                    if inclusive:
+                        return lines[:lineno + 1]
+                    else:
+                        if lineno == 0:
+                            return []
+                        else:
+                            return lines[:lineno]
+
+        return lines
+
+    def prepend_filter(self, lines, location=None):
+        # type: (List[unicode], Any) -> List[unicode]
+        prepend = self.options.get('prepend')
+        if prepend:
+            lines.insert(0, prepend + '\n')
+
+        return lines
+
+    def append_filter(self, lines, location=None):
+        # type: (List[unicode], Any) -> List[unicode]
+        append = self.options.get('append')
+        if append:
+            lines.append(append + '\n')
+
+        return lines
+
+
 class LiteralInclude(Directive):
     """
     Like ``.. include:: :literal:``, but only warns if the include file is
@@ -190,24 +392,6 @@ class LiteralInclude(Directive):
         'diff': directives.unchanged_required,
     }
 
-    def read_with_encoding(self, filename, document, codec_info, encoding):
-        # type: (unicode, nodes.Node, Any, unicode) -> List
-        try:
-            with codecs.StreamReaderWriter(open(filename, 'rb'), codec_info[2],
-                                           codec_info[3], 'strict') as f:
-                lines = f.readlines()
-                lines = dedent_lines(lines, self.options.get('dedent'))  # type: ignore
-                return lines
-        except (IOError, OSError):
-            return [document.reporter.warning(
-                'Include file %r not found or reading it failed' % filename,
-                line=self.lineno)]
-        except UnicodeError:
-            return [document.reporter.warning(
-                'Encoding %r used for reading included file %r seems to '
-                'be wrong, try giving an :encoding: option' %
-                (encoding, filename))]
-
     def run(self):
         # type: () -> List[nodes.Node]
         document = self.state.document
@@ -215,181 +399,51 @@ class LiteralInclude(Directive):
             return [document.reporter.warning('File insertion disabled',
                                               line=self.lineno)]
         env = document.settings.env
-        rel_filename, filename = env.relfn2path(self.arguments[0])
 
-        if 'pyobject' in self.options and 'lines' in self.options:
-            return [document.reporter.warning(
-                'Cannot use both "pyobject" and "lines" options',
-                line=self.lineno)]
+        # convert options['diff'] to absolute path
+        if 'diff' in self.options:
+            _, path = env.relfn2path(self.options['diff'])
+            self.options['diff'] = path
 
-        if 'lineno-match' in self.options and 'lineno-start' in self.options:
-            return [document.reporter.warning(
-                'Cannot use both "lineno-match" and "lineno-start"',
-                line=self.lineno)]
+        try:
+            location = self.state_machine.get_source_and_line(self.lineno)
+            rel_filename, filename = env.relfn2path(self.arguments[0])
+            env.note_dependency(rel_filename)
 
-        if 'lineno-match' in self.options and \
-           (set(['append', 'prepend']) & set(self.options.keys())):
-            return [document.reporter.warning(
-                'Cannot use "lineno-match" and "append" or "prepend"',
-                line=self.lineno)]
+            reader = LiteralIncludeReader(filename, self.options, env.config)
+            text, lines = reader.read(location=location)
 
-        if 'start-after' in self.options and 'start-at' in self.options:
-            return [document.reporter.warning(
-                'Cannot use both "start-after" and "start-at" options',
-                line=self.lineno)]
+            retnode = nodes.literal_block(text, text, source=filename)
+            set_source_info(self, retnode)
+            if self.options.get('diff'):  # if diff is set, set udiff
+                retnode['language'] = 'udiff'
+            elif 'language' in self.options:
+                retnode['language'] = self.options['language']
+            retnode['linenos'] = ('linenos' in self.options or
+                                  'lineno-start' in self.options or
+                                  'lineno-match' in self.options)
+            retnode['classes'] += self.options.get('class', [])
+            extra_args = retnode['highlight_args'] = {}
+            if 'empahsize-lines' in self.options:
+                hl_lines = parselinenos(self.options['emphasize-lines'], lines)
+                if any(i >= lines for i in hl_lines):
+                    logger.warning('line number spec is out of range(1-%d): %r' %
+                                   (lines, self.options['emphasize_lines']),
+                                   location=location)
+                extra_args['hl_lines'] = [x + 1 for x in hl_lines if x < lines]
+            extra_args['linenostart'] = reader.lineno_start
 
-        if 'end-before' in self.options and 'end-at' in self.options:
-            return [document.reporter.warning(
-                'Cannot use both "end-before" and "end-at" options',
-                line=self.lineno)]
-
-        encoding = self.options.get('encoding', env.config.source_encoding)
-        codec_info = codecs.lookup(encoding)
-
-        lines = self.read_with_encoding(filename, document,
-                                        codec_info, encoding)
-        if lines and not isinstance(lines[0], string_types):
-            return lines
-
-        diffsource = self.options.get('diff')
-        if diffsource is not None:
-            tmp, fulldiffsource = env.relfn2path(diffsource)
-
-            difflines = self.read_with_encoding(fulldiffsource, document,
-                                                codec_info, encoding)
-            if not isinstance(difflines[0], string_types):
-                return difflines
-            diff = unified_diff(
-                difflines,
-                lines,
-                diffsource,
-                self.arguments[0])
-            lines = list(diff)
-
-        linenostart = self.options.get('lineno-start', 1)
-        if 'lineno-match' in self.options:
-            linenostart = 1
-        objectname = self.options.get('pyobject')
-        if objectname is not None:
-            from sphinx.pycode import ModuleAnalyzer
-            analyzer = ModuleAnalyzer.for_file(filename, '')
-            tags = analyzer.find_tags()
-            if objectname not in tags:
-                return [document.reporter.warning(
-                    'Object named %r not found in include file %r' %
-                    (objectname, filename), line=self.lineno)]
-            else:
-                lines = lines[tags[objectname][1] - 1: tags[objectname][2] - 1]
-                if 'lineno-match' in self.options:
-                    linenostart = tags[objectname][1]
-
-        linespec = self.options.get('emphasize-lines')
-        if linespec:
-            try:
-                hl_lines = [x + 1 for x in parselinenos(linespec, len(lines))]
-            except ValueError as err:
-                return [document.reporter.warning(str(err), line=self.lineno)]
-        else:
-            hl_lines = None
-
-        start_str = self.options.get('start-after')
-        start_inclusive = False
-        if self.options.get('start-at') is not None:
-            start_str = self.options.get('start-at')
-            start_inclusive = True
-        end_str = self.options.get('end-before')
-        end_inclusive = False
-        if self.options.get('end-at') is not None:
-            end_str = self.options.get('end-at')
-            end_inclusive = True
-        if start_str is not None or end_str is not None:
-            use = not start_str
-            res = []
-            for line_number, line in enumerate(lines):
-                if not use and start_str and start_str in line:
-                    if start_inclusive:
-                        res.append(line)
-                    if 'lineno-match' in self.options:
-                        linenostart = line_number + 1
-                        if not start_inclusive:
-                            linenostart += 1
-                    use = True
-                elif use and end_str and end_str in line:
-                    if end_inclusive:
-                        res.append(line)
-                    break
-                elif use:
-                    res.append(line)
-            lines = res
-
-        linespec = self.options.get('lines')
-        if linespec:
-            try:
-                linelist = parselinenos(linespec, len(lines))
-            except ValueError as err:
-                return [document.reporter.warning(str(err), line=self.lineno)]
-
-            if 'lineno-match' in self.options:
-                # make sure the line list is not "disjoint".
-                previous = linelist[0]
-                for line_number in linelist[1:]:
-                    if line_number == previous + 1:
-                        previous = line_number
-                        continue
-                    return [document.reporter.warning(
-                        'Cannot use "lineno-match" with a disjoint set of '
-                        '"lines"', line=self.lineno)]
-                linenostart += linelist[0]
-            # just ignore non-existing lines
-            lines = [lines[i] for i in linelist if i < len(lines)]
-            if not lines:
-                return [document.reporter.warning(
-                    'Line spec %r: no lines pulled from include file %r' %
-                    (linespec, filename), line=self.lineno)]
-
-        prepend = self.options.get('prepend')
-        if prepend:
-            lines.insert(0, prepend + '\n')
-
-        append = self.options.get('append')
-        if append:
-            lines.append(append + '\n')
-
-        text = ''.join(lines)
-        if self.options.get('tab-width'):
-            text = text.expandtabs(self.options['tab-width'])
-        retnode = nodes.literal_block(text, text, source=filename)
-        set_source_info(self, retnode)
-        if diffsource:  # if diff is set, set udiff
-            retnode['language'] = 'udiff'
-        if 'language' in self.options:
-            retnode['language'] = self.options['language']
-        retnode['linenos'] = 'linenos' in self.options or \
-                             'lineno-start' in self.options or \
-                             'lineno-match' in self.options
-        retnode['classes'] += self.options.get('class', [])
-        extra_args = retnode['highlight_args'] = {}
-        if hl_lines is not None:
-            extra_args['hl_lines'] = hl_lines
-        extra_args['linenostart'] = linenostart
-        env.note_dependency(rel_filename)
-
-        caption = self.options.get('caption')
-        if caption is not None:
-            if not caption:
-                caption = self.arguments[0]
-            try:
+            if 'caption' in self.options:
+                caption = self.options['caption'] or self.arguments[0]
                 retnode = container_wrapper(self, retnode, caption)
-            except ValueError as exc:
-                document = self.state.document
-                errmsg = _('Invalid caption: %s' % exc[0][0].astext())  # type: ignore
-                return [document.reporter.warning(errmsg, line=self.lineno)]
 
-        # retnode will be note_implicit_target that is linked from caption and numref.
-        # when options['name'] is provided, it should be primary ID.
-        self.add_name(retnode)
+            # retnode will be note_implicit_target that is linked from caption and numref.
+            # when options['name'] is provided, it should be primary ID.
+            self.add_name(retnode)
 
-        return [retnode]
+            return [retnode]
+        except Exception as exc:
+            return [document.reporter.warning(str(exc), line=self.lineno)]
 
 
 def setup(app):

sphinx/util/__init__.py

@@ -392,16 +392,18 @@ def parselinenos(spec, total):
             elif len(begend) == 1:
                 items.append(int(begend[0]) - 1)
             elif len(begend) == 2:
-                start, end = begend
-                start = start or 1  # type: ignore
-                                    # left half open (cf. -10)
-                end = end or total  # type: ignore
-                                    # right half open (cf. 10-)
-                items.extend(range(int(start) - 1, int(end)))
+                start = int(begend[0] or 1)     # type: ignore
+                                                # left half open (cf. -10)
+                end = int(begend[1] or max(start, total))   # type: ignore
+                                                            # right half open (cf. 10-)
+                if start > end:  # invalid range (cf. 10-1)
+                    raise ValueError
+                items.extend(range(start - 1, end))
             else:
                 raise ValueError
         except Exception:
             raise ValueError('invalid line number spec: %r' % spec)
+
     return items
 
 

tests/roots/test-directive-code/dedent.rst

@@ -1,35 +0,0 @@
-Dedent
-======
-
-Literal Include
----------------
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 10-11
-   :dedent: 0
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 10-11
-   :dedent: 1
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 10-11
-   :dedent: 2
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 10-11
-   :dedent: 3
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 10-11
-   :dedent: 4
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 10-11
-   :dedent: 1000

tests/roots/test-directive-code/dedent_code.rst

@@ -1,53 +0,0 @@
-Dedent
-======
-
-Code blocks
------------
-
-.. code-block:: ruby
-   :linenos:
-   :dedent: 0
-
-       def ruby?
-           false
-       end
-
-.. code-block:: ruby
-   :linenos:
-   :dedent: 1
-
-       def ruby?
-           false
-       end
-
-.. code-block:: ruby
-   :linenos:
-   :dedent: 2
-
-       def ruby?
-           false
-       end
-
-.. code-block:: ruby
-   :linenos:
-   :dedent: 3
-
-       def ruby?
-           false
-       end
-
-.. code-block:: ruby
-   :linenos:
-   :dedent: 4
-
-       def ruby?
-           false
-       end
-
-.. code-block:: ruby
-   :linenos:
-   :dedent: 1000
-
-       def ruby?
-           false
-       end

tests/roots/test-directive-code/lineno_match.rst

@@ -1,32 +0,0 @@
-Literal Includes with Line Numbers Matching
-===========================================
-
-.. literalinclude:: literal.inc
-   :language: python
-   :pyobject: Foo
-   :lineno-match:
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lines: 5-6,7,8-9
-   :lineno-match:
-
-.. literalinclude:: literal.inc
-   :language: python
-   :start-after: pass
-   :lineno-match:
-
-.. literalinclude:: literal.inc
-   :language: python
-   :start-after: Literally
-   :lines: 1,2
-   :lineno-match:
-
-.. literalinclude:: literal.inc
-   :language: python
-   :start-at: class Bar:
-   :end-at: pass
-   :lineno-match:
-
-.. literalinclude:: empty.inc
-   :lineno-match:

tests/roots/test-directive-code/lineno_start.rst

@@ -1,6 +0,0 @@
-Literal Includes with Line Numbers Starting from 200
-====================================================
-
-.. literalinclude:: literal.inc
-   :language: python
-   :lineno-start: 200

tests/roots/test-directive-code/linenos.rst

@@ -4,3 +4,15 @@ Literal Includes with Line Numbers
 .. literalinclude:: literal.inc
    :language: python
    :linenos:
+
+.. literalinclude:: literal.inc
+   :language: python
+   :lineno-start: 200
+
+.. literalinclude:: literal.inc
+   :language: python
+   :lines: 5-9
+   :lineno-match:
+
+.. literalinclude:: empty.inc
+   :lineno-match:

tests/roots/test-directive-code/literal-diff.inc

@@ -0,0 +1,14 @@
+# Literally included file using Python highlighting
+# -*- coding: utf-8 -*-
+
+foo = "Including Unicode characters: üöä"
+
+class Foo:
+    pass
+
+class Bar:
+    def baz(self):
+        pass
+
+# comment after Bar class
+def bar(): pass

tests/roots/test-directive-code/target.py

@@ -20,3 +20,8 @@ def block_start_with_comment():
 def block_start_with_blank():
 
     return 1
+
+
+class Qux:
+	def quux(self):
+		pass

tests/test_directive_code.py

@@ -11,7 +11,234 @@
 
 import pytest
 
-from util import etree_parse
+from sphinx.config import Config
+from sphinx.directives.code import LiteralIncludeReader
+from util import etree_parse, rootdir
+
+TESTROOT_PATH = rootdir / 'roots' / 'test-directive-code'
+LITERAL_INC_PATH = TESTROOT_PATH / 'literal.inc'
+DUMMY_CONFIG = Config(None, None, {}, '')
+
+
+def test_LiteralIncludeReader():
+    options = {'lineno-match': True}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == LITERAL_INC_PATH.text()
+    assert lines == 14
+    assert reader.lineno_start == 1
+
+
+def test_LiteralIncludeReader_lineno_start():
+    options = {'lineno-start': 5}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == LITERAL_INC_PATH.text()
+    assert lines == 14
+    assert reader.lineno_start == 5
+
+
+def test_LiteralIncludeReader_pyobject1():
+    options = {'lineno-match': True, 'pyobject': 'Foo'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("class Foo:\n"
+                       "    pass\n")
+    assert reader.lineno_start == 6
+
+
+def test_LiteralIncludeReader_pyobject2():
+    options = {'pyobject': 'Bar'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("class Bar:\n"
+                       "    def baz():\n"
+                       "        pass\n")
+    assert reader.lineno_start == 1  # no lineno-match
+
+
+def test_LiteralIncludeReader_pyobject3():
+    options = {'pyobject': 'Bar.baz'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("    def baz():\n"
+                       "        pass\n")
+
+
+def test_LiteralIncludeReader_pyobject_and_lines():
+    options = {'pyobject': 'Bar', 'lines': '2-'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("    def baz():\n"
+                       "        pass\n")
+
+
+def test_LiteralIncludeReader_lines1():
+    options = {'lines': '1-4'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == (u"# Literally included file using Python highlighting\n"
+                       u"# -*- coding: utf-8 -*-\n"
+                       u"\n"
+                       u"foo = \"Including Unicode characters: üöä\"\n")
+
+
+def test_LiteralIncludeReader_lines2():
+    options = {'lines': '1,4,6'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == (u"# Literally included file using Python highlighting\n"
+                       u"foo = \"Including Unicode characters: üöä\"\n"
+                       u"class Foo:\n")
+
+
+def test_LiteralIncludeReader_lines_and_lineno_match1():
+    options = {'lines': '4-6', 'lineno-match': True}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == (u"foo = \"Including Unicode characters: üöä\"\n"
+                       u"\n"
+                       u"class Foo:\n")
+    assert reader.lineno_start == 4
+
+
+@pytest.mark.sphinx()  # init locale for errors
+def test_LiteralIncludeReader_lines_and_lineno_match2(app, status, warning):
+    options = {'lines': '1,4,6', 'lineno-match': True}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    with pytest.raises(ValueError):
+        content, lines = reader.read()
+
+
+@pytest.mark.sphinx()  # init locale for errors
+def test_LiteralIncludeReader_lines_and_lineno_match3(app, status, warning):
+    options = {'lines': '100-', 'lineno-match': True}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    with pytest.raises(ValueError):
+        content, lines = reader.read()
+
+
+def test_LiteralIncludeReader_start_at():
+    options = {'lineno-match': True, 'start-at': 'Foo', 'end-at': 'Bar'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("class Foo:\n"
+                       "    pass\n"
+                       "\n"
+                       "class Bar:\n")
+    assert reader.lineno_start == 6
+
+
+def test_LiteralIncludeReader_start_after():
+    options = {'lineno-match': True, 'start-after': 'Foo', 'end-before': 'Bar'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("    pass\n"
+                       "\n")
+    assert reader.lineno_start == 7
+
+
+def test_LiteralIncludeReader_start_after_and_lines():
+    options = {'lineno-match': True, 'lines': '6-',
+               'start-after': 'coding', 'end-before': 'comment'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("\n"
+                       "class Bar:\n"
+                       "    def baz():\n"
+                       "        pass\n"
+                       "\n")
+    assert reader.lineno_start == 8
+
+
+def test_LiteralIncludeReader_start_at_and_lines():
+    options = {'lines': '2, 3, 5', 'start-at': 'foo', 'end-before': '#'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("\n"
+                       "class Foo:\n"
+                       "\n")
+    assert reader.lineno_start == 1
+
+
+def test_LiteralIncludeReader_prepend():
+    options = {'lines': '1', 'prepend': 'Hello', 'append': 'Sphinx'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("Hello\n"
+                       "# Literally included file using Python highlighting\n"
+                       "Sphinx\n")
+
+
+def test_LiteralIncludeReader_dedent():
+    # dedent: 2
+    options = {'lines': '10-12', 'dedent': 2}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("  def baz():\n"
+                       "      pass\n"
+                       "\n")
+
+    # dedent: 4
+    options = {'lines': '10-12', 'dedent': 4}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("def baz():\n"
+                       "    pass\n"
+                       "\n")
+
+    # dedent: 6
+    options = {'lines': '10-12', 'dedent': 6}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("f baz():\n"
+                       "  pass\n"
+                       "\n")
+
+
+def test_LiteralIncludeReader_tabwidth():
+    # tab-width: 4
+    options = {'tab-width': 4, 'pyobject': 'Qux'}
+    reader = LiteralIncludeReader(TESTROOT_PATH / 'target.py', options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("class Qux:\n"
+                       "    def quux(self):\n"
+                       "        pass\n")
+
+    # tab-width: 8
+    options = {'tab-width': 8, 'pyobject': 'Qux'}
+    reader = LiteralIncludeReader(TESTROOT_PATH / 'target.py', options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("class Qux:\n"
+                       "        def quux(self):\n"
+                       "                pass\n")
+
+
+def test_LiteralIncludeReader_tabwidth_dedent():
+    options = {'tab-width': 4, 'dedent': 4, 'pyobject': 'Qux.quux'}
+    reader = LiteralIncludeReader(TESTROOT_PATH / 'target.py', options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("def quux(self):\n"
+                       "    pass\n")
+
+
+def test_LiteralIncludeReader_diff():
+    options = {'diff': TESTROOT_PATH / 'literal-diff.inc'}
+    reader = LiteralIncludeReader(LITERAL_INC_PATH, options, DUMMY_CONFIG)
+    content, lines = reader.read()
+    assert content == ("--- " + TESTROOT_PATH + "/literal-diff.inc\n"
+                       "+++ " + TESTROOT_PATH + "/literal.inc\n"
+                       "@@ -7,8 +7,8 @@\n"
+                       "     pass\n"
+                       " \n"
+                       " class Bar:\n"
+                       "-    def baz(self):\n"
+                       "+    def baz():\n"
+                       "         pass\n"
+                       " \n"
+                       "-# comment after Bar class\n"
+                       "+# comment after Bar class definition\n"
+                       " def bar(): pass\n")
 
 
 @pytest.mark.sphinx('xml', testroot='directive-code')
@@ -30,25 +257,6 @@ def test_code_block(app, status, warning):
     assert actual == expect
 
 
-@pytest.mark.sphinx('xml', testroot='directive-code')
-def test_code_block_dedent(app, status, warning):
-    app.builder.build(['dedent_code'])
-    et = etree_parse(app.outdir / 'dedent_code.xml')
-    blocks = et.findall('./section/section/literal_block')
-
-    for i in range(5):  # 0-4
-        actual = blocks[i].text
-        indent = " " * (4 - i)
-        expect = (
-            indent + "def ruby?\n" +
-            indent + "    false\n" +
-            indent + "end"
-        )
-        assert (i, actual) == (i, expect)
-
-    assert blocks[5].text == '\n\n'   # dedent: 1000
-
-
 @pytest.mark.sphinx('html', testroot='directive-code')
 def test_code_block_caption_html(app, status, warning):
     app.builder.build(['caption'])
@@ -103,24 +311,6 @@ def test_literal_include(app, status, warning):
     assert actual == literal_src
 
 
-@pytest.mark.sphinx('xml', testroot='directive-code')
-def test_literal_include_dedent(app, status, warning):
-    literal_src = (app.srcdir / 'literal.inc').text(encoding='utf-8')
-    literal_lines = [l[4:] for l in literal_src.split('\n')[9:11]]
-
-    app.builder.build(['dedent'])
-    et = etree_parse(app.outdir / 'dedent.xml')
-    blocks = et.findall('./section/section/literal_block')
-
-    for i in range(5):  # 0-4
-        actual = blocks[i].text
-        indent = ' ' * (4 - i)
-        expect = '\n'.join(indent + l for l in literal_lines) + '\n'
-        assert (i, actual) == (i, expect)
-
-    assert blocks[5].text == '\n\n'   # dedent: 1000
-
-
 @pytest.mark.sphinx('xml', testroot='directive-code')
 def test_literal_include_block_start_with_comment_or_brank(app, status, warning):
     app.builder.build(['python'])
@@ -149,91 +339,48 @@ def test_literal_include_block_start_with_comment_or_brank(app, status, warning)
 def test_literal_include_linenos(app, status, warning):
     app.builder.build(['linenos'])
     html = (app.outdir / 'linenos.html').text(encoding='utf-8')
-    linenos = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        ' 1\n'
-        ' 2\n'
-        ' 3\n'
-        ' 4\n'
-        ' 5\n'
-        ' 6\n'
-        ' 7\n'
-        ' 8\n'
-        ' 9\n'
-        '10\n'
-        '11\n'
-        '12\n'
-        '13\n'
-        '14</pre></div></td>')
-    assert linenos in html
 
+    # :linenos:
+    assert ('<td class="linenos"><div class="linenodiv"><pre>'
+            ' 1\n'
+            ' 2\n'
+            ' 3\n'
+            ' 4\n'
+            ' 5\n'
+            ' 6\n'
+            ' 7\n'
+            ' 8\n'
+            ' 9\n'
+            '10\n'
+            '11\n'
+            '12\n'
+            '13\n'
+            '14</pre></div></td>' in html)
 
-@pytest.mark.sphinx('html', testroot='directive-code')
-def test_literal_include_lineno_start(app, status, warning):
-    app.builder.build(['lineno_start'])
-    html = (app.outdir / 'lineno_start.html').text(encoding='utf-8')
-    linenos = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        '200\n'
-        '201\n'
-        '202\n'
-        '203\n'
-        '204\n'
-        '205\n'
-        '206\n'
-        '207\n'
-        '208\n'
-        '209\n'
-        '210\n'
-        '211\n'
-        '212\n'
-        '213</pre></div></td>')
-    assert linenos in html
+    # :lineno-start:
+    assert ('<td class="linenos"><div class="linenodiv"><pre>'
+            '200\n'
+            '201\n'
+            '202\n'
+            '203\n'
+            '204\n'
+            '205\n'
+            '206\n'
+            '207\n'
+            '208\n'
+            '209\n'
+            '210\n'
+            '211\n'
+            '212\n'
+            '213</pre></div></td>' in html)
 
-
-@pytest.mark.sphinx('html', testroot='directive-code')
-def test_literal_include_lineno_match(app, status, warning):
-    app.builder.build(['lineno_match'])
-    html = (app.outdir / 'lineno_match.html').text(encoding='utf-8')
-    pyobject = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        '6\n'
-        '7</pre></div></td>')
-
-    assert pyobject in html
-
-    lines = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        '5\n'
-        '6\n'
-        '7\n'
-        '8\n'
-        '9</pre></div></td>')
-    assert lines in html
-
-    start_after = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        ' 8\n'
-        ' 9\n'
-        '10\n'
-        '11\n'
-        '12\n'
-        '13\n'
-        '14</pre></div></td>')
-    assert start_after in html
-
-    start_after_with_lines = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        '2\n'
-        '3</pre></div></td>')
-    assert start_after_with_lines in html
-
-    start_at_end_at = (
-        '<td class="linenos"><div class="linenodiv"><pre>'
-        ' 9\n'
-        '10\n'
-        '11</pre></div></td>')
-    assert start_at_end_at in html
+    # :lineno-match:
+    assert ('<td class="linenos"><div class="linenodiv"><pre>'
+            '5\n'
+            '6\n'
+            '7\n'
+            '8\n'
+            '9</pre></div></td>' in html)
 
 
 @pytest.mark.sphinx('latex', testroot='directive-code')

tests/test_util.py

@@ -105,9 +105,13 @@ def test_parselinenos():
     assert parselinenos('7-9', 10) == [6, 7, 8]
     assert parselinenos('7-', 10) == [6, 7, 8, 9]
     assert parselinenos('1,7-', 10) == [0, 6, 7, 8, 9]
+    assert parselinenos('7-7', 10) == [6]
+    assert parselinenos('11-', 10) == [10]
     with pytest.raises(ValueError):
         parselinenos('1-2-3', 10)
     with pytest.raises(ValueError):
         parselinenos('abc-def', 10)
     with pytest.raises(ValueError):
         parselinenos('-', 10)
+    with pytest.raises(ValueError):
+        parselinenos('3-1', 10)