From 18daa75b7fc1f55b4970a91e5395e284301f751d Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 26 Jan 2016 10:55:47 +0900 Subject: [PATCH 01/12] Update CHANGES (ref: #2264) --- CHANGES | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 29c03329a..908e52ed3 100644 --- a/CHANGES +++ b/CHANGES @@ -19,7 +19,8 @@ Bugs fixed * #2243: Ignore strange docstring types for classes, do not crash * #2247: Fix #2205 breaks make html for definition list with classifiers that contains regular-expression like string -* #1565: Show warning if Pygments throws an ErrorToken +* #1565: Sphinx will now emit a warning that highlighting was skipped if the syntax + is incorrect for `code-block`, `literalinclude` and so on. * #2211: Fix paragraphs in table cell doesn't work in Latex output * #2253: ``:pyobject:`` option of ``literalinclude`` directive can't detect indented body block when the block starts with blank or comment lines. From 9b2111514660c0b01bec280393c9e3720be504d0 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 27 Jan 2016 19:38:57 +0900 Subject: [PATCH 02/12] Fix #2265: babel is used in spite of disabling it on ``latex_elements`` --- CHANGES | 2 ++ sphinx/writers/latex.py | 10 +++++----- tests/test_build_latex.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 908e52ed3..0db03d333 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,8 @@ Release 1.3.6 (in development) Bugs fixed ---------- +* #2265: Fix babel is used in spite of disabling it on ``latex_elements`` + Release 1.3.5 (released Jan 24, 2016) ===================================== diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 21ab1549b..367041ece 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -346,11 +346,6 @@ class LaTeXTranslator(nodes.NodeVisitor): return '\\usepackage{%s}' % (packagename,) usepackages = (declare_package(*p) for p in builder.usepackages) self.elements['usepackages'] += "\n".join(usepackages) - if getattr(document.settings, 'contentsname', None): - self.elements['contentsname'] = \ - self.babel_renewcommand(builder, '\\contentsname', - document.settings.contentsname) - self.elements['numfig_format'] = self.generate_numfig_format(builder) # allow the user to override them all self.elements.update(builder.config.latex_elements) if self.elements['extraclassoptions']: @@ -363,6 +358,11 @@ class LaTeXTranslator(nodes.NodeVisitor): else: self.elements['tocdepth'] = ('\\setcounter{tocdepth}{%d}' % (document['tocdepth'] - 1)) + if getattr(document.settings, 'contentsname', None): + self.elements['contentsname'] = \ + self.babel_renewcommand(builder, '\\contentsname', + document.settings.contentsname) + self.elements['numfig_format'] = self.generate_numfig_format(builder) self.highlighter = highlighting.PygmentsBridge( 'latex', diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 1957128e8..4b67bcf8b 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -267,6 +267,25 @@ def test_numref_with_language_ja(app, status, warning): assert '\\hyperref[baz:code22]{Code-\\ref{baz:code22}}' in result +@with_app(buildername='latex', testroot='numfig', + confoverrides={'numfig': True, 'language': 'ru', 'latex_elements': {'babel': ''}}) +def test_numref_on_bable_disabled(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'Python.tex').text(encoding='utf8') + print(result) + print(status.getvalue()) + print(warning.getvalue()) + assert '\\renewcommand{\\figurename}{Fig. }' in result + assert '\\renewcommand{\\tablename}{Table }' in result + assert '\\SetupFloatingEnvironment{literal-block}{name=Listing }' in result + assert '\\hyperref[index:fig1]{Fig. \\ref{index:fig1}}' in result + assert '\\hyperref[baz:fig22]{Figure\\ref{baz:fig22}}' in result + assert '\\hyperref[index:table-1]{Table \\ref{index:table-1}}' in result + assert '\\hyperref[baz:table22]{Table:\\ref{baz:table22}}' in result + assert '\\hyperref[index:code-1]{Listing \\ref{index:code-1}}' in result + assert '\\hyperref[baz:code22]{Code-\\ref{baz:code22}}' in result + + @with_app(buildername='latex') def test_latex_add_latex_package(app, status, warning): app.add_latex_package('foo') From 8740030e8019fe604a4d000490d32d2e3bded291 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 30 Jan 2016 16:14:14 +0900 Subject: [PATCH 03/12] Fix documentation for 'mo auto build". --- doc/intl.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/intl.rst b/doc/intl.rst index bd4f9e889..02f5d08a7 100644 --- a/doc/intl.rst +++ b/doc/intl.rst @@ -106,17 +106,23 @@ This section describe a easy way to translate with sphinx-intl. #. Translate your po files under `./locale//LC_MESSAGES/`. -#. Build mo files and make translated document. +#. make translated document. You need a :confval:`language` parameter in ``conf.py`` or you may also specify the parameter on the command line:: - $ sphinx-intl build $ make -e SPHINXOPTS="-D language='de'" html Congratulations! You got the translated documentation in the ``_build/html`` directory. +.. versionadded:: 1.3 + + sphinx-build that is invoked by make command will build po files into mo files. + + If you are using 1.2.x or earlier, please invoke ``sphinx-intl build`` command + before make command. + Translating ^^^^^^^^^^^ From c6289c311f2c4561b284b1cb29704ced9a5fd86f Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 30 Jan 2016 16:37:58 +0900 Subject: [PATCH 04/12] Fix documentation for 'mo auto build". --- doc/intl.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/intl.rst b/doc/intl.rst index 02f5d08a7..2290b0fde 100644 --- a/doc/intl.rst +++ b/doc/intl.rst @@ -257,9 +257,8 @@ easy to fetch and push translations. ... Done. - Build po files into mo and make html:: + Invoke make html:: - $ sphinx-intl build $ make -e SPHINXOPTS="-D language='de'" html From 1714747318b731a39e0c8104ac53611a47ae202e Mon Sep 17 00:00:00 2001 From: "Kacper Kowalik (Xarthisius)" Date: Tue, 2 Feb 2016 14:16:47 -0600 Subject: [PATCH 05/12] Use ensuredir() for creating directories within BuildEnvironment.read_doc --- sphinx/environment.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 8d0789cfa..9e5a870a4 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -40,7 +40,7 @@ from sphinx import addnodes from sphinx.util import url_re, get_matching_docs, docname_join, split_into, \ FilenameUniqDict, get_figtype, import_object, split_index_msg from sphinx.util.nodes import clean_astext, make_refnode, WarningStream, is_translatable -from sphinx.util.osutil import SEP, getcwd, fs_encoding +from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir from sphinx.util.i18n import find_catalog_files from sphinx.util.console import bold, purple from sphinx.util.matching import compile_matchers @@ -855,9 +855,7 @@ class BuildEnvironment: # save the parsed doctree doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree') - dirname = path.dirname(doctree_filename) - if not path.isdir(dirname): - os.makedirs(dirname) + ensuredir(path.dirname(doctree_filename)) f = open(doctree_filename, 'wb') try: pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL) From bdc230b1ccc189989789f82a55023f369410309d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Thu, 4 Feb 2016 19:58:21 +0100 Subject: [PATCH 06/12] Closes #2295: Avoid mutating dictionary errors while enumerating members in autodoc with Python 3 --- CHANGES | 2 ++ sphinx/ext/autodoc.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 0db03d333..6a7bc2a8e 100644 --- a/CHANGES +++ b/CHANGES @@ -5,6 +5,8 @@ Bugs fixed ---------- * #2265: Fix babel is used in spite of disabling it on ``latex_elements`` +* #2295: Avoid mutating dictionary errors while enumerating members in autodoc + with Python 3 Release 1.3.5 (released Jan 24, 2016) diff --git a/sphinx/ext/autodoc.py b/sphinx/ext/autodoc.py index f906d9666..6f9e057aa 100644 --- a/sphinx/ext/autodoc.py +++ b/sphinx/ext/autodoc.py @@ -606,7 +606,7 @@ class Documenter(object): members = [] else: members = [(mname, self.get_attr(self.object, mname, None)) - for mname in obj_dict.keys()] + for mname in list(obj_dict.keys())] membernames = set(m[0] for m in members) # add instance attributes from the analyzer for aname in analyzed_member_names: From 9b958b6dccf14346fd6a1cd083abe02940d040cb Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 7 Feb 2016 19:34:40 +0900 Subject: [PATCH 07/12] Fix #2291: pdflatex "Counter too large" error from footnotes inside tables of contents --- CHANGES | 1 + sphinx/util/nodes.py | 5 +++-- sphinx/writers/latex.py | 14 +++++++++----- tests/roots/test-footnotes/index.rst | 3 +++ tests/test_build_latex.py | 21 +++++++++++++++++++++ 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 6a7bc2a8e..03e8ebe62 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,7 @@ Bugs fixed * #2265: Fix babel is used in spite of disabling it on ``latex_elements`` * #2295: Avoid mutating dictionary errors while enumerating members in autodoc with Python 3 +* #2291: Fix pdflatex "Counter too large" error from footnotes inside tables of contents Release 1.3.5 (released Jan 24, 2016) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index f8d2d0a93..ccea95777 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -141,9 +141,10 @@ def find_source_node(node): return pnode.source -def traverse_parent(node): +def traverse_parent(node, cls=None): while node: - yield node + if cls is None or isinstance(node, cls): + yield node node = node.parent diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 367041ece..434d8a699 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -25,7 +25,7 @@ from sphinx import highlighting from sphinx.errors import SphinxError from sphinx.locale import admonitionlabels, _ from sphinx.util import split_into -from sphinx.util.nodes import clean_astext +from sphinx.util.nodes import clean_astext, traverse_parent from sphinx.util.osutil import ustrftime from sphinx.util.texescape import tex_escape_map, tex_replace_map from sphinx.util.smartypants import educate_quotes_latex @@ -159,11 +159,15 @@ class ShowUrlsTransform(object): if node.astext() != uri: index = node.parent.index(node) if show_urls == 'footnote': - footnote_nodes = self.create_footnote(uri) - for i, fn in enumerate(footnote_nodes): - node.parent.insert(index + i + 1, fn) + if list(traverse_parent(node, nodes.topic)): + # should not expand references in topics + pass + else: + footnote_nodes = self.create_footnote(uri) + for i, fn in enumerate(footnote_nodes): + node.parent.insert(index + i + 1, fn) - self.expanded = True + self.expanded = True else: # all other true values (b/w compat) textnode = nodes.Text(" (%s)" % uri) node.parent.insert(index + 1, textnode) diff --git a/tests/roots/test-footnotes/index.rst b/tests/roots/test-footnotes/index.rst index a20052037..228fccf8d 100644 --- a/tests/roots/test-footnotes/index.rst +++ b/tests/roots/test-footnotes/index.rst @@ -2,6 +2,9 @@ test-footenotes =============== +.. contents:: + :local: + The section with a reference to [AuthorYear]_ ============================================= diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 4b67bcf8b..4533ee4c0 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -372,6 +372,13 @@ def test_latex_show_urls_is_inline(app, status, warning): print(result) print(status.getvalue()) print(warning.getvalue()) + assert ('\\phantomsection\\label{index:id26}{\\hyperref[index:the\\string-section' + '\\string-with\\string-a\\string-reference\\string-to\\string-authoryear]' + '{\\emph{The section with a reference to \\phantomsection\\label{index:id1}' + '{\\hyperref[index:authoryear]{\\emph{{[}AuthorYear{]}}}}}}}' in result) + assert ('\\phantomsection\\label{index:id27}{\\hyperref[index:the\\string-section' + '\\string-with\\string-a\\string-reference\\string-to]{\\emph{The section ' + 'with a reference to }}}' in result) assert 'First footnote: \\footnote[2]{\nFirst\n}' in result assert 'Second footnote: \\footnote[1]{\nSecond\n}' in result assert '\\href{http://sphinx-doc.org/}{Sphinx} (http://sphinx-doc.org/)' in result @@ -398,6 +405,13 @@ def test_latex_show_urls_is_footnote(app, status, warning): print(result) print(status.getvalue()) print(warning.getvalue()) + assert ('\\phantomsection\\label{index:id26}{\\hyperref[index:the\\string-section' + '\\string-with\\string-a\\string-reference\\string-to\\string-authoryear]' + '{\\emph{The section with a reference to \\phantomsection\\label{index:id1}' + '{\\hyperref[index:authoryear]{\\emph{{[}AuthorYear{]}}}}}}}' in result) + assert ('\\phantomsection\\label{index:id27}{\\hyperref[index:the\\string-section' + '\\string-with\\string-a\\string-reference\\string-to]{\\emph{The section ' + 'with a reference to }}}' in result) assert 'First footnote: \\footnote[2]{\nFirst\n}' in result assert 'Second footnote: \\footnote[1]{\nSecond\n}' in result assert ('\\href{http://sphinx-doc.org/}{Sphinx}' @@ -426,6 +440,13 @@ def test_latex_show_urls_is_no(app, status, warning): print(result) print(status.getvalue()) print(warning.getvalue()) + assert ('\\phantomsection\\label{index:id26}{\\hyperref[index:the\\string-section' + '\\string-with\\string-a\\string-reference\\string-to\\string-authoryear]' + '{\\emph{The section with a reference to \\phantomsection\\label{index:id1}' + '{\\hyperref[index:authoryear]{\\emph{{[}AuthorYear{]}}}}}}}' in result) + assert ('\\phantomsection\\label{index:id27}{\\hyperref[index:the\\string-section' + '\\string-with\\string-a\\string-reference\\string-to]{\\emph{The section ' + 'with a reference to }}}' in result) assert 'First footnote: \\footnote[2]{\nFirst\n}' in result assert 'Second footnote: \\footnote[1]{\nSecond\n}' in result assert '\\href{http://sphinx-doc.org/}{Sphinx}' in result From 9ba7cc815c247205ded53824da99d74d411ca827 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 7 Feb 2016 22:58:14 +0900 Subject: [PATCH 08/12] Fix #2292: Some footnotes disappear from LaTeX output --- CHANGES | 1 + sphinx/writers/latex.py | 11 ++++++++--- tests/roots/test-footnotes/bar.rst | 6 ++++++ tests/roots/test-footnotes/baz.rst | 6 ++++++ tests/roots/test-footnotes/index.rst | 5 +++++ tests/test_build_latex.py | 26 ++++++++++++++++---------- 6 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 tests/roots/test-footnotes/bar.rst create mode 100644 tests/roots/test-footnotes/baz.rst diff --git a/CHANGES b/CHANGES index 03e8ebe62..1be7c2c43 100644 --- a/CHANGES +++ b/CHANGES @@ -8,6 +8,7 @@ Bugs fixed * #2295: Avoid mutating dictionary errors while enumerating members in autodoc with Python 3 * #2291: Fix pdflatex "Counter too large" error from footnotes inside tables of contents +* #2292: Fix some footnotes disappear from LaTeX output Release 1.3.5 (released Jan 24, 2016) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 434d8a699..31189c7da 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -199,9 +199,14 @@ class ShowUrlsTransform(object): def is_auto_footnote(node): return isinstance(node, nodes.footnote) and node.get('auto') - def footnote_ref_by(ids): + def footnote_ref_by(node): + ids = node['ids'] + parent = list(traverse_parent(node, (nodes.document, addnodes.start_of_file)))[0] + def is_footnote_ref(node): - return isinstance(node, nodes.footnote_reference) and ids[0] == node['refid'] + return (isinstance(node, nodes.footnote_reference) and + ids[0] == node['refid'] and + parent in list(traverse_parent(node))) return is_footnote_ref @@ -220,7 +225,7 @@ class ShowUrlsTransform(object): footnote['names'].remove(old_label) footnote['names'].append(label) - for footnote_ref in self.document.traverse(footnote_ref_by(footnote['ids'])): + for footnote_ref in self.document.traverse(footnote_ref_by(footnote)): footnote_ref.remove(footnote_ref[0]) footnote_ref += nodes.Text(label) diff --git a/tests/roots/test-footnotes/bar.rst b/tests/roots/test-footnotes/bar.rst new file mode 100644 index 000000000..660c66306 --- /dev/null +++ b/tests/roots/test-footnotes/bar.rst @@ -0,0 +1,6 @@ +bar +=== + +Same footnote number [1]_ in bar.rst + +.. [1] footnote in bar diff --git a/tests/roots/test-footnotes/baz.rst b/tests/roots/test-footnotes/baz.rst new file mode 100644 index 000000000..af496c5f6 --- /dev/null +++ b/tests/roots/test-footnotes/baz.rst @@ -0,0 +1,6 @@ +baz +=== + +Auto footnote number [#]_ in baz.rst + +.. [#] footnote in baz diff --git a/tests/roots/test-footnotes/index.rst b/tests/roots/test-footnotes/index.rst index 228fccf8d..3a8bc25c5 100644 --- a/tests/roots/test-footnotes/index.rst +++ b/tests/roots/test-footnotes/index.rst @@ -2,6 +2,11 @@ test-footenotes =============== +.. toctree:: + + bar + baz + .. contents:: :local: diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 4533ee4c0..c4a887d44 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -372,6 +372,8 @@ def test_latex_show_urls_is_inline(app, status, warning): print(result) print(status.getvalue()) print(warning.getvalue()) + assert 'Same footnote number \\footnote[1]{\nfootnote in bar\n} in bar.rst' in result + assert 'Auto footnote number \\footnote[1]{\nfootnote in baz\n} in baz.rst' in result assert ('\\phantomsection\\label{index:id26}{\\hyperref[index:the\\string-section' '\\string-with\\string-a\\string-reference\\string-to\\string-authoryear]' '{\\emph{The section with a reference to \\phantomsection\\label{index:id1}' @@ -405,6 +407,8 @@ def test_latex_show_urls_is_footnote(app, status, warning): print(result) print(status.getvalue()) print(warning.getvalue()) + assert 'Same footnote number \\footnote[1]{\nfootnote in bar\n} in bar.rst' in result + assert 'Auto footnote number \\footnote[2]{\nfootnote in baz\n} in baz.rst' in result assert ('\\phantomsection\\label{index:id26}{\\hyperref[index:the\\string-section' '\\string-with\\string-a\\string-reference\\string-to\\string-authoryear]' '{\\emph{The section with a reference to \\phantomsection\\label{index:id1}' @@ -412,20 +416,20 @@ def test_latex_show_urls_is_footnote(app, status, warning): assert ('\\phantomsection\\label{index:id27}{\\hyperref[index:the\\string-section' '\\string-with\\string-a\\string-reference\\string-to]{\\emph{The section ' 'with a reference to }}}' in result) - assert 'First footnote: \\footnote[2]{\nFirst\n}' in result + assert 'First footnote: \\footnote[3]{\nFirst\n}' in result assert 'Second footnote: \\footnote[1]{\nSecond\n}' in result assert ('\\href{http://sphinx-doc.org/}{Sphinx}' - '\\footnote[3]{\nhttp://sphinx-doc.org/\n}' in result) - assert 'Third footnote: \\footnote[5]{\nThird\n}' in result + '\\footnote[4]{\nhttp://sphinx-doc.org/\n}' in result) + assert 'Third footnote: \\footnote[6]{\nThird\n}' in result assert ('\\href{http://sphinx-doc.org/~test/}{URL including tilde}' - '\\footnote[4]{\nhttp://sphinx-doc.org/\\textasciitilde{}test/\n}' in result) - assert ('\\item[{\\href{http://sphinx-doc.org/}{URL in term}\\protect\\footnotemark[7]}] ' - '\\leavevmode\\footnotetext[7]{\nhttp://sphinx-doc.org/\n}\nDescription' in result) - assert ('\\item[{Footnote in term \\protect\\footnotemark[9]}] ' - '\\leavevmode\\footnotetext[9]{\nFootnote in term\n}\nDescription' in result) - assert ('\\item[{\\href{http://sphinx-doc.org/}{Term in deflist}\\protect' - '\\footnotemark[8]}] ' + '\\footnote[5]{\nhttp://sphinx-doc.org/\\textasciitilde{}test/\n}' in result) + assert ('\\item[{\\href{http://sphinx-doc.org/}{URL in term}\\protect\\footnotemark[8]}] ' '\\leavevmode\\footnotetext[8]{\nhttp://sphinx-doc.org/\n}\nDescription' in result) + assert ('\\item[{Footnote in term \\protect\\footnotemark[10]}] ' + '\\leavevmode\\footnotetext[10]{\nFootnote in term\n}\nDescription' in result) + assert ('\\item[{\\href{http://sphinx-doc.org/}{Term in deflist}\\protect' + '\\footnotemark[9]}] ' + '\\leavevmode\\footnotetext[9]{\nhttp://sphinx-doc.org/\n}\nDescription' in result) assert ('\\href{https://github.com/sphinx-doc/sphinx}' '{https://github.com/sphinx-doc/sphinx}\n' in result) assert ('\\href{mailto:sphinx-dev@googlegroups.com}' @@ -440,6 +444,8 @@ def test_latex_show_urls_is_no(app, status, warning): print(result) print(status.getvalue()) print(warning.getvalue()) + assert 'Same footnote number \\footnote[1]{\nfootnote in bar\n} in bar.rst' in result + assert 'Auto footnote number \\footnote[1]{\nfootnote in baz\n} in baz.rst' in result assert ('\\phantomsection\\label{index:id26}{\\hyperref[index:the\\string-section' '\\string-with\\string-a\\string-reference\\string-to\\string-authoryear]' '{\\emph{The section with a reference to \\phantomsection\\label{index:id1}' From aefab514e2bbce775a00c1aaae057044a02033fb Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Wed, 3 Feb 2016 08:57:54 +0900 Subject: [PATCH 09/12] Fix #2287: `sphinx.transforms.Locale` always uses rst parser. Sphinx i18n feature should support parsers that specified source_parsers. --- CHANGES | 2 + sphinx/application.py | 3 +- sphinx/environment.py | 83 ++--------------------------- sphinx/io.py | 121 ++++++++++++++++++++++++++++++++++++++++++ sphinx/quickstart.py | 1 + sphinx/transforms.py | 62 +++++++++++----------- 6 files changed, 161 insertions(+), 111 deletions(-) create mode 100644 sphinx/io.py diff --git a/CHANGES b/CHANGES index 1be7c2c43..f01c3eae9 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,8 @@ Bugs fixed with Python 3 * #2291: Fix pdflatex "Counter too large" error from footnotes inside tables of contents * #2292: Fix some footnotes disappear from LaTeX output +* #2287: ``sphinx.transforms.Locale`` always uses rst parser. Sphinx i18n feature should + support parsers that specified source_parsers. Release 1.3.5 (released Jan 24, 2016) diff --git a/sphinx/application.py b/sphinx/application.py index 957a1be26..07c3024dd 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -35,7 +35,8 @@ from sphinx.errors import SphinxError, SphinxWarning, ExtensionError, \ from sphinx.domains import ObjType, BUILTIN_DOMAINS from sphinx.domains.std import GenericObject, Target, StandardDomain from sphinx.builders import BUILTIN_BUILDERS -from sphinx.environment import BuildEnvironment, SphinxStandaloneReader +from sphinx.environment import BuildEnvironment +from sphinx.io import SphinxStandaloneReader from sphinx.util import pycompat # noqa: imported for side-effects from sphinx.util import import_object from sphinx.util.tags import Tags diff --git a/sphinx/environment.py b/sphinx/environment.py index 9e5a870a4..a039ba071 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -23,22 +23,21 @@ from os import path from glob import glob from itertools import groupby -from six import iteritems, itervalues, text_type, class_types, string_types, next +from six import iteritems, itervalues, text_type, class_types, next from six.moves import cPickle as pickle from docutils import nodes -from docutils.io import FileInput, NullOutput +from docutils.io import NullOutput from docutils.core import Publisher from docutils.utils import Reporter, relative_path, get_source_line -from docutils.readers import standalone from docutils.parsers.rst import roles, directives from docutils.parsers.rst.languages import en as english from docutils.parsers.rst.directives.html import MetaBody -from docutils.writers import UnfilteredWriter from docutils.frontend import OptionParser from sphinx import addnodes +from sphinx.io import SphinxStandaloneReader, SphinxDummyWriter, SphinxFileInput from sphinx.util import url_re, get_matching_docs, docname_join, split_into, \ - FilenameUniqDict, get_figtype, import_object, split_index_msg + FilenameUniqDict, get_figtype, split_index_msg from sphinx.util.nodes import clean_astext, make_refnode, WarningStream, is_translatable from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir from sphinx.util.i18n import find_catalog_files @@ -49,12 +48,7 @@ from sphinx.util.websupport import is_commentable from sphinx.errors import SphinxError, ExtensionError from sphinx.locale import _ from sphinx.versioning import add_uids, merge_doctrees -from sphinx.transforms import ( - DefaultSubstitutions, MoveModuleTargets, ApplySourceWorkaround, - HandleCodeBlocks, AutoNumbering, SortIds, CitationReferences, Locale, - RemoveTranslatableInline, SphinxContentsFilter, ExtraTranslatableNodes, -) - +from sphinx.transforms import SphinxContentsFilter orig_role_function = roles.role orig_directive_function = directives.directive @@ -97,73 +91,6 @@ class NoUri(Exception): pass -class SphinxStandaloneReader(standalone.Reader): - """ - Add our own transforms. - """ - transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, Locale, CitationReferences, - DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, - AutoNumbering, SortIds, RemoveTranslatableInline] - - def __init__(self, parsers={}, *args, **kwargs): - standalone.Reader.__init__(self, *args, **kwargs) - self.parser_map = {} - for suffix, parser_class in parsers.items(): - if isinstance(parser_class, string_types): - parser_class = import_object(parser_class, 'source parser') - self.parser_map[suffix] = parser_class() - - def read(self, source, parser, settings): - self.source = source - - for suffix in self.parser_map: - if source.source_path.endswith(suffix): - self.parser = self.parser_map[suffix] - break - - if not self.parser: - self.parser = parser - self.settings = settings - self.input = self.source.read() - self.parse() - return self.document - - def get_transforms(self): - return standalone.Reader.get_transforms(self) + self.transforms - - -class SphinxDummyWriter(UnfilteredWriter): - supported = ('html',) # needed to keep "meta" nodes - - def translate(self): - pass - - -class SphinxFileInput(FileInput): - def __init__(self, app, env, *args, **kwds): - self.app = app - self.env = env - kwds['error_handler'] = 'sphinx' # py3: handle error on open. - FileInput.__init__(self, *args, **kwds) - - def decode(self, data): - if isinstance(data, text_type): # py3: `data` already decoded. - return data - return data.decode(self.encoding, 'sphinx') # py2: decoding - - def read(self): - data = FileInput.read(self) - if self.app: - arg = [data] - self.app.emit('source-read', self.env.docname, arg) - data = arg[0] - if self.env.config.rst_epilog: - data = data + '\n' + self.env.config.rst_epilog + '\n' - if self.env.config.rst_prolog: - data = self.env.config.rst_prolog + '\n' + data - return data - - class BuildEnvironment: """ The environment in which the ReST files are translated. diff --git a/sphinx/io.py b/sphinx/io.py new file mode 100644 index 000000000..aee6336ae --- /dev/null +++ b/sphinx/io.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +""" + sphinx.io + ~~~~~~~~~ + + Input/Output files + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +from docutils.io import FileInput +from docutils.readers import standalone +from docutils.writers import UnfilteredWriter +from six import string_types, text_type + +from sphinx.transforms import ApplySourceWorkaround, ExtraTranslatableNodes, Locale, \ + CitationReferences, DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, \ + AutoNumbering, SortIds, RemoveTranslatableInline +from sphinx.util import import_object + + +class SphinxBaseReader(standalone.Reader): + """ + Add our source parsers + """ + def __init__(self, parsers={}, *args, **kwargs): + standalone.Reader.__init__(self, *args, **kwargs) + self.parser_map = {} + for suffix, parser_class in parsers.items(): + if isinstance(parser_class, string_types): + parser_class = import_object(parser_class, 'source parser') + self.parser_map[suffix] = parser_class() + + def read(self, source, parser, settings): + self.source = source + + for suffix in self.parser_map: + if source.source_path.endswith(suffix): + self.parser = self.parser_map[suffix] + break + + if not self.parser: + self.parser = parser + self.settings = settings + self.input = self.source.read() + self.parse() + return self.document + + def get_transforms(self): + return standalone.Reader.get_transforms(self) + self.transforms + + +class SphinxStandaloneReader(SphinxBaseReader): + """ + Add our own transforms. + """ + transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, Locale, CitationReferences, + DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, + AutoNumbering, SortIds, RemoveTranslatableInline] + + +class SphinxI18nReader(SphinxBaseReader): + """ + Replacer for document.reporter.get_source_and_line method. + + reST text lines for translation do not have the original source line number. + This class provides the correct line numbers when reporting. + """ + + transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, CitationReferences, + DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, + AutoNumbering, SortIds, RemoveTranslatableInline] + + def __init__(self, *args, **kwargs): + SphinxBaseReader.__init__(self, *args, **kwargs) + self.lineno = None + + def set_lineno_for_reporter(self, lineno): + self.lineno = lineno + + def new_document(self): + document = SphinxBaseReader.new_document(self) + reporter = document.reporter + + def get_source_and_line(lineno=None): + return reporter.source, self.lineno + + reporter.get_source_and_line = get_source_and_line + return document + + +class SphinxDummyWriter(UnfilteredWriter): + supported = ('html',) # needed to keep "meta" nodes + + def translate(self): + pass + + +class SphinxFileInput(FileInput): + def __init__(self, app, env, *args, **kwds): + self.app = app + self.env = env + kwds['error_handler'] = 'sphinx' # py3: handle error on open. + FileInput.__init__(self, *args, **kwds) + + def decode(self, data): + if isinstance(data, text_type): # py3: `data` already decoded. + return data + return data.decode(self.encoding, 'sphinx') # py2: decoding + + def read(self): + data = FileInput.read(self) + if self.app: + arg = [data] + self.app.emit('source-read', self.env.docname, arg) + data = arg[0] + if self.env.config.rst_epilog: + data = data + '\n' + self.env.config.rst_epilog + '\n' + if self.env.config.rst_prolog: + data = self.env.config.rst_prolog + '\n' + data + return data diff --git a/sphinx/quickstart.py b/sphinx/quickstart.py index 9a60785b4..dd5f8b2ca 100644 --- a/sphinx/quickstart.py +++ b/sphinx/quickstart.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for details. """ from __future__ import print_function +from __future__ import absolute_import import re import os diff --git a/sphinx/transforms.py b/sphinx/transforms.py index 4619449e7..bfae7f812 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -12,8 +12,8 @@ from os import path from docutils import nodes -from docutils.utils import new_document, relative_path -from docutils.parsers.rst import Parser as RSTParser +from docutils.io import StringInput +from docutils.utils import relative_path from docutils.transforms import Transform from docutils.transforms.parts import ContentsFilter @@ -202,21 +202,33 @@ class ExtraTranslatableNodes(Transform): node['translatable'] = True -class CustomLocaleReporter(object): +def publish_msgstr(source, source_path, source_line, config, settings): + """Publish msgstr (single line) into docutils document + + :param unicode source: source text + :param unicode source_path: source path for warning indication + :param source_line: source line for warning indication + :param sphinx.config.Config config: sphinx config + :param docutils.frontend.Values settings: docutils settings + :return: document + :rtype: docutils.nodes.document """ - Replacer for document.reporter.get_source_and_line method. - - reST text lines for translation do not have the original source line number. - This class provides the correct line numbers when reporting. - """ - def __init__(self, source, line): - self.source, self.line = source, line - - def set_reporter(self, document): - document.reporter.get_source_and_line = self.get_source_and_line - - def get_source_and_line(self, lineno=None): - return self.source, self.line + from sphinx.io import SphinxI18nReader + reader = SphinxI18nReader( + parsers=config.source_parsers, + parser_name='restructuredtext', # default parser + ) + reader.set_lineno_for_reporter(source_line) + doc = reader.read( + source=StringInput(source=source, source_path=source_path), + parser=reader.parser, + settings=settings, + ) + try: + doc = doc[0] + except IndexError: # empty node + pass + return doc class Locale(Transform): @@ -244,8 +256,6 @@ class Locale(Transform): if not has_catalog: return - parser = RSTParser() - # phase1: replace reference ids with translated names for node, msg in extract_messages(self.document): msgstr = catalog.gettext(msg) @@ -267,13 +277,7 @@ class Locale(Transform): if isinstance(node, LITERAL_TYPE_NODES): msgstr = '::\n\n' + indent(msgstr, ' '*3) - patch = new_document(source, settings) - CustomLocaleReporter(node.source, node.line).set_reporter(patch) - parser.parse(msgstr, patch) - try: - patch = patch[0] - except IndexError: # empty node - pass + patch = publish_msgstr(msgstr, source, node.line, env.config, settings) # XXX doctest and other block markup if not isinstance(patch, nodes.paragraph): continue # skip for now @@ -390,13 +394,7 @@ class Locale(Transform): if isinstance(node, LITERAL_TYPE_NODES): msgstr = '::\n\n' + indent(msgstr, ' '*3) - patch = new_document(source, settings) - CustomLocaleReporter(node.source, node.line).set_reporter(patch) - parser.parse(msgstr, patch) - try: - patch = patch[0] - except IndexError: # empty node - pass + patch = publish_msgstr(msgstr, source, node.line, env.config, settings) # XXX doctest and other block markup if not isinstance( patch, From c22f5beb752ce13582000327b80bb33a7121c398 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Sat, 6 Jun 2015 13:12:56 -0700 Subject: [PATCH 10/12] Add suffix to HTML context. Fixes #1873 --- doc/templating.rst | 5 +++++ sphinx/builders/html.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/doc/templating.rst b/doc/templating.rst index 19ed14840..5286e0312 100644 --- a/doc/templating.rst +++ b/doc/templating.rst @@ -399,3 +399,8 @@ are in HTML form), these variables are also available: * ``includehidden`` (``False`` by default): if true, the TOC tree will also contain hidden entries. + +.. data:: page_source_suffix + + The suffix of the file that was rendered. Since we support a list of :confval:`source_suffix`, + this will allow you to properly link to the original source file. diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 8dcb905d3..e247f3327 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -408,6 +408,9 @@ class StandaloneHTMLBuilder(Builder): # metadata for the document meta = self.env.metadata.get(docname) + # Suffix for the document + source_suffix = '.' + self.env.doc2path(docname).split('.')[-1] + # local TOC and global TOC tree self_toc = self.env.get_toc_for(docname, self) toc = self.render_partial(self_toc)['fragment'] @@ -425,6 +428,7 @@ class StandaloneHTMLBuilder(Builder): toc = toc, # only display a TOC if there's more than one item to show display_toc = (self.env.toc_num_entries[docname] > 1), + page_source_suffix = source_suffix, ) def write_doc(self, docname, doctree): From f29edef477de6284ae94cbaaf66525ad8ee4c418 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Tue, 9 Feb 2016 18:09:43 +0900 Subject: [PATCH 11/12] Update CHANGES for #1873, #1876, #2278 --- CHANGES | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES b/CHANGES index f01c3eae9..805ea45c1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,13 @@ Release 1.3.6 (in development) ============================== +Features added +-------------- + +* #1873, #1876, #2278: Add ``page_source_suffix`` html context variable. This should be + introduced with :confval:`source_parsers` feature. Thanks for Eric Holscher. + + Bugs fixed ---------- From 906d1c905d1fe74a5efb7493eb42f974b97c150d Mon Sep 17 00:00:00 2001 From: GunWoo Choi Date: Fri, 13 Nov 2015 12:52:56 +0900 Subject: [PATCH 12/12] Make sure to iterate obj_dict's keys using list, not view In Python3 `dict.keys()` returns a view object not a list cherry-picked from master 22ce010e0 to stable for #2116. --- sphinx/ext/autodoc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/autodoc.py b/sphinx/ext/autodoc.py index 6f9e057aa..8c7524aaa 100644 --- a/sphinx/ext/autodoc.py +++ b/sphinx/ext/autodoc.py @@ -17,7 +17,8 @@ import inspect import traceback from types import FunctionType, BuiltinFunctionType, MethodType -from six import iteritems, itervalues, text_type, class_types, string_types +from six import iterkeys, iteritems, itervalues, text_type, class_types, \ + string_types from docutils import nodes from docutils.utils import assemble_option_dict from docutils.statemachine import ViewList @@ -598,7 +599,7 @@ class Documenter(object): # __dict__ contains only the members directly defined in # the class (but get them via getattr anyway, to e.g. get # unbound method objects instead of function objects); - # using keys() because apparently there are objects for which + # using list(iterkeys()) because apparently there are objects for which # __dict__ changes while getting attributes try: obj_dict = self.get_attr(self.object, '__dict__') @@ -606,7 +607,7 @@ class Documenter(object): members = [] else: members = [(mname, self.get_attr(self.object, mname, None)) - for mname in list(obj_dict.keys())] + for mname in list(iterkeys(obj_dict))] membernames = set(m[0] for m in members) # add instance attributes from the analyzer for aname in analyzed_member_names: