From 17cd06f237e78eab7e8201c9e98deab6a3ca9304 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 26 Jun 2016 22:22:25 +0900 Subject: [PATCH 1/7] Add sphinx.util.fileutil.copy_asset_file() --- sphinx/builders/html.py | 1 + sphinx/util/__init__.py | 14 ++------- sphinx/util/fileutil.py | 39 +++++++++++++++++++++++++ tests/test_util_fileutil.py | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 sphinx/util/fileutil.py create mode 100644 tests/test_util_fileutil.py diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 27e961bd9..1f1811778 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -1069,6 +1069,7 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder): self.theme = None # no theme necessary self.templates = None # no template bridge necessary self.init_translator_class() + self.init_templates() self.init_highlighter() def get_target_uri(self, docname, typ=None): diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 0c823fb1a..e1a65d950 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -18,7 +18,7 @@ import posixpath import traceback import unicodedata from os import path -from codecs import open, BOM_UTF8 +from codecs import BOM_UTF8 from collections import deque from six import iteritems, text_type, binary_type @@ -32,6 +32,7 @@ import jinja2 import sphinx from sphinx.errors import PycodeError, SphinxParallelError, ExtensionError from sphinx.util.console import strip_colors +from sphinx.util.fileutil import copy_asset_file from sphinx.util.osutil import fs_encoding # import other utilities; partly for backwards compatibility, so don't @@ -158,16 +159,7 @@ def copy_static_entry(source, targetdir, builder, context={}, if matcher(relpath): return if path.isfile(source): - target = path.join(targetdir, path.basename(source)) - if source.lower().endswith('_t') and builder.templates: - # templated! - fsrc = open(source, 'r', encoding='utf-8') - fdst = open(target[:-2], 'w', encoding='utf-8') - fdst.write(builder.templates.render_string(fsrc.read(), context)) - fsrc.close() - fdst.close() - else: - copyfile(source, target) + copy_asset_file(source, targetdir, context, builder.templates) elif path.isdir(source): if not path.isdir(targetdir): os.mkdir(targetdir) diff --git a/sphinx/util/fileutil.py b/sphinx/util/fileutil.py new file mode 100644 index 000000000..df168b495 --- /dev/null +++ b/sphinx/util/fileutil.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" + sphinx.util.fileutil + ~~~~~~~~~~~~~~~~~~~~ + + File utility functions for Sphinx. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import os +import codecs +from sphinx.util.osutil import copyfile + + +def copy_asset_file(source, destination, context={}, renderer=None): + """Copy an asset file to destination. + + On copying, it expands the template variables if the asset is a template file. + + :param source: The path to source file + :param destination: The path to destination file or directory + :param context: The template variables + :param renderer: The template engine + """ + if os.path.exists(destination) and os.path.isdir(destination): + # Use source filename if destination points a directory + destination = os.path.join(destination, os.path.basename(source)) + + if source.lower().endswith('_t'): + if renderer is None: + msg = 'Template engine is not initialized. Failed to render %s' % source + raise RuntimeError(msg) + + with codecs.open(source, 'r', encoding='utf-8') as fsrc: + with codecs.open(destination[:-2], 'w', encoding='utf-8') as fdst: + fdst.write(renderer.render_string(fsrc.read(), context)) + else: + copyfile(source, destination) diff --git a/tests/test_util_fileutil.py b/tests/test_util_fileutil.py new file mode 100644 index 000000000..c5680c890 --- /dev/null +++ b/tests/test_util_fileutil.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" + test_util_fileutil + ~~~~~~~~~~~~~~~~~~ + + Tests sphinx.util.fileutil functions. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +from sphinx.util.fileutil import copy_asset_file +from sphinx.jinja2glue import BuiltinTemplateLoader + +from mock import Mock +from util import with_tempdir + + +class DummyTemplateLoader(BuiltinTemplateLoader): + def __init__(self): + BuiltinTemplateLoader.__init__(self) + builder = Mock() + builder.config.templates_path = [] + builder.app.translater = None + self.init(builder) + + +@with_tempdir +def test_copy_asset_file(tmpdir): + renderer = DummyTemplateLoader() + + # copy normal file + src = (tmpdir / 'asset.txt') + src.write_text('# test data') + dest = (tmpdir / 'output.txt') + + copy_asset_file(src, dest) + assert dest.exists() + assert src.text() == dest.text() + + # copy template file + src = (tmpdir / 'asset.txt_t') + src.write_text('# {{var1}} data') + dest = (tmpdir / 'output.txt_t') + + copy_asset_file(src, dest, {'var1': 'template'}, renderer) + assert not dest.exists() + assert (tmpdir / 'output.txt').exists() + assert (tmpdir / 'output.txt').text() == '# template data' + + # copy template file to subdir + src = (tmpdir / 'asset.txt_t') + src.write_text('# {{var1}} data') + subdir = (tmpdir / 'subdir') + subdir.makedirs() + + copy_asset_file(src, subdir, {'var1': 'template'}, renderer) + assert (subdir / 'asset.txt').exists() + assert (subdir / 'asset.txt').text() == '# template data' From 07ddff99335a34cad9e917b35a2ad81a4ceb4b2e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 27 Jun 2016 11:54:42 +0900 Subject: [PATCH 2/7] Add sphinx.util.fileutil.copy_asset() --- sphinx/util/fileutil.py | 35 ++++++++++++++++++++++++++- tests/test_util_fileutil.py | 47 ++++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/sphinx/util/fileutil.py b/sphinx/util/fileutil.py index df168b495..1f6bede1a 100644 --- a/sphinx/util/fileutil.py +++ b/sphinx/util/fileutil.py @@ -10,7 +10,9 @@ """ import os import codecs -from sphinx.util.osutil import copyfile +import posixpath +from docutils.utils import relative_path +from sphinx.util.osutil import copyfile, ensuredir, walk def copy_asset_file(source, destination, context={}, renderer=None): @@ -37,3 +39,34 @@ def copy_asset_file(source, destination, context={}, renderer=None): fdst.write(renderer.render_string(fsrc.read(), context)) else: copyfile(source, destination) + + +def copy_asset(source, destination, excluded=lambda path: False, context={}, renderer=None): + """Copy asset files to destination recursively. + + On copying, it expands the template variables if the asset is a template file. + + :param source: The path to source file or directory + :param destination: The path to destination directory + :param excluded: The matcher to determine the given path should be copied or not + :param context: The template variables + :param renderer: The template engine + """ + ensuredir(destination) + if os.path.isfile(source): + copy_asset_file(source, destination, context, renderer) + return + + for root, dirs, files in walk(source): + reldir = relative_path(source, root) + for dir in dirs[:]: + if excluded(posixpath.join(reldir, dir)): + dirs.remove(dir) + else: + ensuredir(posixpath.join(destination, reldir, dir)) + + for filename in files: + if not excluded(posixpath.join(reldir, filename)): + copy_asset_file(posixpath.join(root, filename), + posixpath.join(destination, reldir), + context, renderer) diff --git a/tests/test_util_fileutil.py b/tests/test_util_fileutil.py index c5680c890..5e13d79b2 100644 --- a/tests/test_util_fileutil.py +++ b/tests/test_util_fileutil.py @@ -8,7 +8,7 @@ :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from sphinx.util.fileutil import copy_asset_file +from sphinx.util.fileutil import copy_asset, copy_asset_file from sphinx.jinja2glue import BuiltinTemplateLoader from mock import Mock @@ -56,3 +56,48 @@ def test_copy_asset_file(tmpdir): copy_asset_file(src, subdir, {'var1': 'template'}, renderer) assert (subdir / 'asset.txt').exists() assert (subdir / 'asset.txt').text() == '# template data' + + +@with_tempdir +def test_copy_asset(tmpdir): + renderer = DummyTemplateLoader() + + # prepare source files + source = (tmpdir / 'source') + source.makedirs() + (source / 'index.rst').write_text('index.rst') + (source / 'foo.rst_t').write_text('{{var1}}.rst') + (source / '_static').makedirs() + (source / '_static' / 'basic.css').write_text('basic.css') + (source / '_templates').makedirs() + (source / '_templates' / 'layout.html').write_text('layout.html') + (source / '_templates' / 'sidebar.html_t').write_text('sidebar: {{var2}}') + + # copy a single file + assert not (tmpdir / 'test1').exists() + copy_asset(source / 'index.rst', tmpdir / 'test1') + assert (tmpdir / 'test1').exists() + assert (tmpdir / 'test1/index.rst').exists() + + # copy directories + destdir = tmpdir / 'test2' + copy_asset(source, destdir, context=dict(var1='bar', var2='baz'), renderer=renderer) + assert (destdir / 'index.rst').exists() + assert (destdir / 'foo.rst').exists() + assert (destdir / 'foo.rst').text() == 'bar.rst' + assert (destdir / '_static' / 'basic.css').exists() + assert (destdir / '_templates' / 'layout.html').exists() + assert (destdir / '_templates' / 'sidebar.html').exists() + assert (destdir / '_templates' / 'sidebar.html').text() == 'sidebar: baz' + + # copy with exclusion + def excluded(path): + return ('sidebar.html' in path or 'basic.css' in path) + + destdir = tmpdir / 'test3' + copy_asset(source, destdir, excluded, renderer=renderer) + assert (destdir / 'index.rst').exists() + assert (destdir / 'foo.rst').exists() + assert not (destdir / '_static' / 'basic.css').exists() + assert (destdir / '_templates' / 'layout.html').exists() + assert not (destdir / '_templates' / 'sidebar.html').exists() From db23797c57ac448a464da25f85e5b5bfd83eb188 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 27 Jun 2016 12:13:16 +0900 Subject: [PATCH 3/7] copy_asset_file() does not expand if context not given --- sphinx/util/fileutil.py | 16 +++++++++------- tests/test_util_fileutil.py | 23 +++++++++++++++++------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/sphinx/util/fileutil.py b/sphinx/util/fileutil.py index 1f6bede1a..8ab590472 100644 --- a/sphinx/util/fileutil.py +++ b/sphinx/util/fileutil.py @@ -15,21 +15,22 @@ from docutils.utils import relative_path from sphinx.util.osutil import copyfile, ensuredir, walk -def copy_asset_file(source, destination, context={}, renderer=None): +def copy_asset_file(source, destination, context=None, renderer=None): """Copy an asset file to destination. - On copying, it expands the template variables if the asset is a template file. + On copying, it expands the template variables if context argument is given and + the asset is a template file. :param source: The path to source file :param destination: The path to destination file or directory - :param context: The template variables + :param context: The template variables. If not given, template files are simply copied :param renderer: The template engine """ if os.path.exists(destination) and os.path.isdir(destination): # Use source filename if destination points a directory destination = os.path.join(destination, os.path.basename(source)) - if source.lower().endswith('_t'): + if source.lower().endswith('_t') and context: if renderer is None: msg = 'Template engine is not initialized. Failed to render %s' % source raise RuntimeError(msg) @@ -41,15 +42,16 @@ def copy_asset_file(source, destination, context={}, renderer=None): copyfile(source, destination) -def copy_asset(source, destination, excluded=lambda path: False, context={}, renderer=None): +def copy_asset(source, destination, excluded=lambda path: False, context=None, renderer=None): """Copy asset files to destination recursively. - On copying, it expands the template variables if the asset is a template file. + On copying, it expands the template variables if context argument is given and + the asset is a template file. :param source: The path to source file or directory :param destination: The path to destination directory :param excluded: The matcher to determine the given path should be copied or not - :param context: The template variables + :param context: The template variables. If not given, template files are simply copied :param renderer: The template engine """ ensuredir(destination) diff --git a/tests/test_util_fileutil.py b/tests/test_util_fileutil.py index 5e13d79b2..a56543614 100644 --- a/tests/test_util_fileutil.py +++ b/tests/test_util_fileutil.py @@ -50,12 +50,22 @@ def test_copy_asset_file(tmpdir): # copy template file to subdir src = (tmpdir / 'asset.txt_t') src.write_text('# {{var1}} data') - subdir = (tmpdir / 'subdir') - subdir.makedirs() + subdir1 = (tmpdir / 'subdir') + subdir1.makedirs() - copy_asset_file(src, subdir, {'var1': 'template'}, renderer) - assert (subdir / 'asset.txt').exists() - assert (subdir / 'asset.txt').text() == '# template data' + copy_asset_file(src, subdir1, {'var1': 'template'}, renderer) + assert (subdir1 / 'asset.txt').exists() + assert (subdir1 / 'asset.txt').text() == '# template data' + + # copy template file without context + src = (tmpdir / 'asset.txt_t') + subdir2 = (tmpdir / 'subdir2') + subdir2.makedirs() + + copy_asset_file(src, subdir2) + assert not (subdir2 / 'asset.txt').exists() + assert (subdir2 / 'asset.txt_t').exists() + assert (subdir2 / 'asset.txt_t').text() == '# {{var1}} data' @with_tempdir @@ -95,7 +105,8 @@ def test_copy_asset(tmpdir): return ('sidebar.html' in path or 'basic.css' in path) destdir = tmpdir / 'test3' - copy_asset(source, destdir, excluded, renderer=renderer) + copy_asset(source, destdir, excluded, + context=dict(var1='bar', var2='baz'), renderer=renderer) assert (destdir / 'index.rst').exists() assert (destdir / 'foo.rst').exists() assert not (destdir / '_static' / 'basic.css').exists() From 829b5631a34a7da67e6dee18853c82c12e1aa59d Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 3 Jul 2016 15:11:15 +0900 Subject: [PATCH 4/7] Add testcase for compile_matchers() --- tests/test_util_matching.py | 83 +++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/test_util_matching.py diff --git a/tests/test_util_matching.py b/tests/test_util_matching.py new file mode 100644 index 000000000..fe86c3bbc --- /dev/null +++ b/tests/test_util_matching.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" + test_util_matching + ~~~~~~~~~~~~~~~~~~ + + Tests sphinx.util.matching functions. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +from sphinx.util.matching import compile_matchers + + +def test_compile_matchers(): + # exact matching + pat = compile_matchers(['hello.py']).pop() + assert pat('hello.py') + assert not pat('hello-py') + assert not pat('subdir/hello.py') + + # wild card (*) + pat = compile_matchers(['hello.*']).pop() + assert pat('hello.py') + assert pat('hello.rst') + + pat = compile_matchers(['*.py']).pop() + assert pat('hello.py') + assert pat('world.py') + assert not pat('subdir/hello.py') + + # wild card (**) + pat = compile_matchers(['hello.**']).pop() + assert pat('hello.py') + assert pat('hello.rst') + assert pat('hello.py/world.py') + + pat = compile_matchers(['**.py']).pop() + assert pat('hello.py') + assert pat('world.py') + assert pat('subdir/hello.py') + + pat = compile_matchers(['**/hello.py']).pop() + assert not pat('hello.py') + assert pat('subdir/hello.py') + assert pat('subdir/subdir/hello.py') + + # wild card (?) + pat = compile_matchers(['hello.?']).pop() + assert pat('hello.c') + assert not pat('hello.py') + + # pattern ([...]) + pat = compile_matchers(['hello[12\\].py']).pop() + assert pat('hello1.py') + assert pat('hello2.py') + assert pat('hello\\.py') + assert not pat('hello3.py') + + pat = compile_matchers(['hello[^12].py']).pop() # "^" is not negative identifier + assert pat('hello1.py') + assert pat('hello2.py') + assert pat('hello^.py') + assert not pat('hello3.py') + + # negative pattern ([!...]) + pat = compile_matchers(['hello[!12].py']).pop() + assert not pat('hello1.py') + assert not pat('hello2.py') + assert not pat('hello/.py') # negative pattern does not match to "/" + assert pat('hello3.py') + + # non patterns + pat = compile_matchers(['hello[.py']).pop() + assert pat('hello[.py') + assert not pat('hello.py') + + pat = compile_matchers(['hello[].py']).pop() + assert pat('hello[].py') + assert not pat('hello.py') + + pat = compile_matchers(['hello[!].py']).pop() + assert pat('hello[!].py') + assert not pat('hello.py') From bb1e6f9044567fcd05a27bb1cc55f727c86fe2bd Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 3 Jul 2016 15:45:01 +0900 Subject: [PATCH 5/7] Refactor: Replace copy_extra_entry() with copy_asset() --- sphinx/builders/html.py | 16 +++++----- sphinx/util/__init__.py | 31 ------------------- sphinx/util/matching.py | 18 +++++++++++ .../extra/subdir/.htaccess | 0 .../extra/subdir/.htpasswd | 0 tests/test_build_html.py | 2 ++ tests/test_util_matching.py | 10 +++++- 7 files changed, 38 insertions(+), 39 deletions(-) create mode 100644 tests/roots/test-html_extra_path/extra/subdir/.htaccess create mode 100644 tests/roots/test-html_extra_path/extra/subdir/.htpasswd diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 1f1811778..25cdc04bd 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -27,12 +27,13 @@ from docutils.frontend import OptionParser from docutils.readers.doctree import Reader as DoctreeReader from sphinx import package_dir, __display_version__ -from sphinx.util import jsonimpl, copy_static_entry, copy_extra_entry +from sphinx.util import jsonimpl, copy_static_entry from sphinx.util.i18n import format_date from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \ movefile, copyfile from sphinx.util.nodes import inline_all_toctrees -from sphinx.util.matching import patmatch, compile_matchers +from sphinx.util.fileutil import copy_asset +from sphinx.util.matching import patmatch, compile_matchers, Matcher from sphinx.config import string_classes from sphinx.locale import _, l_ from sphinx.search import js_index @@ -650,14 +651,15 @@ class StandaloneHTMLBuilder(Builder): def copy_extra_files(self): # copy html_extra_path files self.info(bold('copying extra files... '), nonl=True) - extraentries = [path.join(self.confdir, epath) - for epath in self.config.html_extra_path] - matchers = compile_matchers(self.config.exclude_patterns) - for entry in extraentries: + excluded = Matcher(self.config.exclude_patterns) + + for extra_path in self.config.html_extra_path: + entry = path.join(self.confdir, extra_path) if not path.exists(entry): self.warn('html_extra_path entry %r does not exist' % entry) continue - copy_extra_entry(entry, self.outdir, matchers) + + copy_asset(entry, self.outdir, excluded) self.info('done') def write_buildinfo(self): diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index e1a65d950..036e57ac0 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -173,37 +173,6 @@ def copy_static_entry(source, targetdir, builder, context={}, builder, context, level=level+1, exclude_matchers=exclude_matchers) - -def copy_extra_entry(source, targetdir, exclude_matchers=()): - """Copy a HTML builder extra_path entry from source to targetdir. - - Handles all possible cases of files, directories and subdirectories. - """ - def excluded(path): - relpath = relative_path(os.path.dirname(source), path) - return any(matcher(relpath) for matcher in exclude_matchers) - - def copy_extra_file(source_, targetdir_): - if not excluded(source_): - target = path.join(targetdir_, os.path.basename(source_)) - copyfile(source_, target) - - if os.path.isfile(source): - copy_extra_file(source, targetdir) - return - - for root, dirs, files in os.walk(source): - reltargetdir = os.path.join(targetdir, relative_path(source, root)) - for dir in dirs[:]: - if excluded(os.path.join(root, dir)): - dirs.remove(dir) - else: - target = os.path.join(reltargetdir, dir) - if not path.exists(target): - os.mkdir(target) - for file in files: - copy_extra_file(os.path.join(root, file), reltargetdir) - _DEBUG_HEADER = '''\ # Sphinx version: %s # Python version: %s (%s) diff --git a/sphinx/util/matching.py b/sphinx/util/matching.py index 91fda6378..8f17980ca 100644 --- a/sphinx/util/matching.py +++ b/sphinx/util/matching.py @@ -62,6 +62,24 @@ def compile_matchers(patterns): return [re.compile(_translate_pattern(pat)).match for pat in patterns] +class Matcher(object): + """A pattern matcher for Multiple shell-style glob patterns. + + Note: this modifies the patterns to work with copy_asset(). + For example, "**/index.rst" matches with "index.rst" + """ + + def __init__(self, patterns): + expanded = [pat[3:] for pat in patterns if pat.startswith('**/')] + self.patterns = compile_matchers(patterns + expanded) + + def __call__(self, string): + return self.match(string) + + def match(self, string): + return any(pat(string) for pat in self.patterns) + + _pat_cache = {} diff --git a/tests/roots/test-html_extra_path/extra/subdir/.htaccess b/tests/roots/test-html_extra_path/extra/subdir/.htaccess new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-html_extra_path/extra/subdir/.htpasswd b/tests/roots/test-html_extra_path/extra/subdir/.htpasswd new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_build_html.py b/tests/test_build_html.py index 173ad6cf5..78ef1770c 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -987,6 +987,8 @@ def test_html_extra_path(app, status, warning): assert (app.outdir / 'rimg.png').exists() assert not (app.outdir / '_build/index.html').exists() assert (app.outdir / 'background.png').exists() + assert (app.outdir / 'subdir' / '.htaccess').exists() + assert not (app.outdir / 'subdir' / '.htpasswd').exists() @with_app(buildername='html', confoverrides={'html_sourcelink_suffix': ''}) diff --git a/tests/test_util_matching.py b/tests/test_util_matching.py index fe86c3bbc..9e99a5322 100644 --- a/tests/test_util_matching.py +++ b/tests/test_util_matching.py @@ -8,7 +8,7 @@ :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from sphinx.util.matching import compile_matchers +from sphinx.util.matching import compile_matchers, Matcher def test_compile_matchers(): @@ -81,3 +81,11 @@ def test_compile_matchers(): pat = compile_matchers(['hello[!].py']).pop() assert pat('hello[!].py') assert not pat('hello.py') + + +def test_Matcher(): + matcher = Matcher(['hello.py', '**/world.py']) + assert matcher('hello.py') + assert not matcher('subdir/hello.py') + assert matcher('world.py') + assert matcher('subdir/world.py') From 02dbf860c2cf530adf85bd7a6325a90e9b3e3ca4 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 3 Jul 2016 16:43:18 +0900 Subject: [PATCH 6/7] Refactor: Use copy_asset() to copy html_static_path --- sphinx/builders/html.py | 13 ++++++------- .../conf.py | 3 +++ .../extra/.htaccess | 0 .../extra/.htpasswd | 0 tests/roots/test-html_assets/extra/API.html_t | 1 + .../extra/css/style.css | 0 .../extra/rimg.png | Bin .../extra/subdir/.htaccess | 0 .../extra/subdir/.htpasswd | 0 .../index.rst | 0 .../static/.htaccess} | 0 .../static/.htpasswd} | 0 tests/roots/test-html_assets/static/API.html_t | 1 + .../test-html_assets/static/css/style.css | 0 .../static/rimg.png} | Bin .../test-html_assets/static/subdir/.htaccess | 0 .../test-html_assets/static/subdir/.htpasswd | 0 .../test-html_assets/subdir/_build/index.html | 0 .../test-html_assets/subdir/background.png | Bin 0 -> 218 bytes tests/test_build_html.py | 17 +++++++++++++++-- 20 files changed, 26 insertions(+), 9 deletions(-) rename tests/roots/{test-html_extra_path => test-html_assets}/conf.py (63%) rename tests/roots/{test-html_extra_path => test-html_assets}/extra/.htaccess (100%) rename tests/roots/{test-html_extra_path => test-html_assets}/extra/.htpasswd (100%) create mode 100644 tests/roots/test-html_assets/extra/API.html_t rename tests/roots/{test-html_extra_path => test-html_assets}/extra/css/style.css (100%) rename tests/roots/{test-html_extra_path => test-html_assets}/extra/rimg.png (100%) rename tests/roots/{test-html_extra_path => test-html_assets}/extra/subdir/.htaccess (100%) rename tests/roots/{test-html_extra_path => test-html_assets}/extra/subdir/.htpasswd (100%) rename tests/roots/{test-html_extra_path => test-html_assets}/index.rst (100%) rename tests/roots/{test-html_extra_path/extra/API.html_t => test-html_assets/static/.htaccess} (100%) rename tests/roots/{test-html_extra_path/subdir/_build/index.html => test-html_assets/static/.htpasswd} (100%) create mode 100644 tests/roots/test-html_assets/static/API.html_t create mode 100644 tests/roots/test-html_assets/static/css/style.css rename tests/roots/{test-html_extra_path/subdir/background.png => test-html_assets/static/rimg.png} (100%) create mode 100644 tests/roots/test-html_assets/static/subdir/.htaccess create mode 100644 tests/roots/test-html_assets/static/subdir/.htpasswd create mode 100644 tests/roots/test-html_assets/subdir/_build/index.html create mode 100644 tests/roots/test-html_assets/subdir/background.png diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 25cdc04bd..9e8600968 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -33,7 +33,7 @@ from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \ movefile, copyfile from sphinx.util.nodes import inline_all_toctrees from sphinx.util.fileutil import copy_asset -from sphinx.util.matching import patmatch, compile_matchers, Matcher +from sphinx.util.matching import patmatch, Matcher from sphinx.config import string_classes from sphinx.locale import _, l_ from sphinx.search import js_index @@ -620,15 +620,14 @@ class StandaloneHTMLBuilder(Builder): copy_static_entry(entry, path.join(self.outdir, '_static'), self, ctx) # then, copy over all user-supplied static files - staticentries = [path.join(self.confdir, spath) - for spath in self.config.html_static_path] - matchers = compile_matchers(self.config.exclude_patterns) - for entry in staticentries: + excluded = Matcher(self.config.exclude_patterns + ["**/.*"]) + for static_path in self.config.html_static_path: + entry = path.join(self.confdir, static_path) if not path.exists(entry): self.warn('html_static_path entry %r does not exist' % entry) continue - copy_static_entry(entry, path.join(self.outdir, '_static'), self, - ctx, exclude_matchers=matchers) + copy_asset(entry, path.join(self.outdir, '_static'), excluded, + context=ctx, renderer=self.templates) # copy logo and favicon files if not already in static path if self.config.html_logo: logobase = path.basename(self.config.html_logo) diff --git a/tests/roots/test-html_extra_path/conf.py b/tests/roots/test-html_assets/conf.py similarity index 63% rename from tests/roots/test-html_extra_path/conf.py rename to tests/roots/test-html_assets/conf.py index 53ee62197..a17e417a3 100644 --- a/tests/roots/test-html_extra_path/conf.py +++ b/tests/roots/test-html_assets/conf.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- master_doc = 'index' +project = 'Sphinx' +version = '1.4.4' +html_static_path = ['static', 'subdir'] html_extra_path = ['extra', 'subdir'] exclude_patterns = ['**/_build', '**/.htpasswd'] diff --git a/tests/roots/test-html_extra_path/extra/.htaccess b/tests/roots/test-html_assets/extra/.htaccess similarity index 100% rename from tests/roots/test-html_extra_path/extra/.htaccess rename to tests/roots/test-html_assets/extra/.htaccess diff --git a/tests/roots/test-html_extra_path/extra/.htpasswd b/tests/roots/test-html_assets/extra/.htpasswd similarity index 100% rename from tests/roots/test-html_extra_path/extra/.htpasswd rename to tests/roots/test-html_assets/extra/.htpasswd diff --git a/tests/roots/test-html_assets/extra/API.html_t b/tests/roots/test-html_assets/extra/API.html_t new file mode 100644 index 000000000..34ecd9df1 --- /dev/null +++ b/tests/roots/test-html_assets/extra/API.html_t @@ -0,0 +1 @@ +{{ project }}-{{ version }} diff --git a/tests/roots/test-html_extra_path/extra/css/style.css b/tests/roots/test-html_assets/extra/css/style.css similarity index 100% rename from tests/roots/test-html_extra_path/extra/css/style.css rename to tests/roots/test-html_assets/extra/css/style.css diff --git a/tests/roots/test-html_extra_path/extra/rimg.png b/tests/roots/test-html_assets/extra/rimg.png similarity index 100% rename from tests/roots/test-html_extra_path/extra/rimg.png rename to tests/roots/test-html_assets/extra/rimg.png diff --git a/tests/roots/test-html_extra_path/extra/subdir/.htaccess b/tests/roots/test-html_assets/extra/subdir/.htaccess similarity index 100% rename from tests/roots/test-html_extra_path/extra/subdir/.htaccess rename to tests/roots/test-html_assets/extra/subdir/.htaccess diff --git a/tests/roots/test-html_extra_path/extra/subdir/.htpasswd b/tests/roots/test-html_assets/extra/subdir/.htpasswd similarity index 100% rename from tests/roots/test-html_extra_path/extra/subdir/.htpasswd rename to tests/roots/test-html_assets/extra/subdir/.htpasswd diff --git a/tests/roots/test-html_extra_path/index.rst b/tests/roots/test-html_assets/index.rst similarity index 100% rename from tests/roots/test-html_extra_path/index.rst rename to tests/roots/test-html_assets/index.rst diff --git a/tests/roots/test-html_extra_path/extra/API.html_t b/tests/roots/test-html_assets/static/.htaccess similarity index 100% rename from tests/roots/test-html_extra_path/extra/API.html_t rename to tests/roots/test-html_assets/static/.htaccess diff --git a/tests/roots/test-html_extra_path/subdir/_build/index.html b/tests/roots/test-html_assets/static/.htpasswd similarity index 100% rename from tests/roots/test-html_extra_path/subdir/_build/index.html rename to tests/roots/test-html_assets/static/.htpasswd diff --git a/tests/roots/test-html_assets/static/API.html_t b/tests/roots/test-html_assets/static/API.html_t new file mode 100644 index 000000000..34ecd9df1 --- /dev/null +++ b/tests/roots/test-html_assets/static/API.html_t @@ -0,0 +1 @@ +{{ project }}-{{ version }} diff --git a/tests/roots/test-html_assets/static/css/style.css b/tests/roots/test-html_assets/static/css/style.css new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-html_extra_path/subdir/background.png b/tests/roots/test-html_assets/static/rimg.png similarity index 100% rename from tests/roots/test-html_extra_path/subdir/background.png rename to tests/roots/test-html_assets/static/rimg.png diff --git a/tests/roots/test-html_assets/static/subdir/.htaccess b/tests/roots/test-html_assets/static/subdir/.htaccess new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-html_assets/static/subdir/.htpasswd b/tests/roots/test-html_assets/static/subdir/.htpasswd new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-html_assets/subdir/_build/index.html b/tests/roots/test-html_assets/subdir/_build/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-html_assets/subdir/background.png b/tests/roots/test-html_assets/subdir/background.png new file mode 100644 index 0000000000000000000000000000000000000000..1081dc1439fb984dfa7ef627afe3c7dc476fdbce GIT binary patch literal 218 zcmeAS@N?(olHy`uVBq!ia0vp^j6iI|!3HFkf4uMuBv2gW?!>U}oXkrghqJ&VvY3H^ zTNs2H8D`Cq01C2~c>21s-(chw7$R|bZ|_0D0|q>YSbqDzW^|HYIk%*-&O)*\na + 1 < b' in content -@with_app(buildername='html', testroot='html_extra_path') -def test_html_extra_path(app, status, warning): +@with_app(buildername='html', testroot='html_assets') +def test_html_assets(app, status, warning): app.builder.build_all() + # html_static_path + assert not (app.outdir / '_static' / '.htaccess').exists() + assert not (app.outdir / '_static' / '.htpasswd').exists() + assert (app.outdir / '_static' / 'API.html').exists() + assert (app.outdir / '_static' / 'API.html').text() == 'Sphinx-1.4.4' + assert (app.outdir / '_static' / 'css/style.css').exists() + assert (app.outdir / '_static' / 'rimg.png').exists() + assert not (app.outdir / '_static' / '_build/index.html').exists() + assert (app.outdir / '_static' / 'background.png').exists() + assert not (app.outdir / '_static' / 'subdir' / '.htaccess').exists() + assert not (app.outdir / '_static' / 'subdir' / '.htpasswd').exists() + + # html_extra_path assert (app.outdir / '.htaccess').exists() assert not (app.outdir / '.htpasswd').exists() assert (app.outdir / 'API.html_t').exists() From f8955b16d6c5b03b95d2a5ee85b739a1de307d2e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 3 Jul 2016 17:16:28 +0900 Subject: [PATCH 7/7] Replace all copy_static_entry() with copy_asset() At same time, copy_static_entry() is now deprecataed --- CHANGES | 2 ++ sphinx/builders/applehelp.py | 14 ++++++-------- sphinx/builders/changes.py | 12 +++++------- sphinx/builders/html.py | 13 ++++++------- sphinx/util/__init__.py | 2 +- sphinx/util/fileutil.py | 6 ++++++ sphinx/util/matching.py | 3 +++ 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/CHANGES b/CHANGES index 1f007c479..c6812b714 100644 --- a/CHANGES +++ b/CHANGES @@ -21,6 +21,8 @@ Incompatible changes * #2454: The filename of sourcelink is now changed. The value of `html_sourcelink_suffix` will be appended to the original filename (like ``index.rst.txt``). +* ``sphinx.util.copy_static_entry()`` is now deprecated. + Use ``sphinx.util.fileutil.copy_asset()`` instead. Features added diff --git a/sphinx/builders/applehelp.py b/sphinx/builders/applehelp.py index 095457b80..e8a476f60 100644 --- a/sphinx/builders/applehelp.py +++ b/sphinx/builders/applehelp.py @@ -18,11 +18,11 @@ import shlex from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.config import string_classes -from sphinx.util import copy_static_entry from sphinx.util.osutil import copyfile, ensuredir, make_filename from sphinx.util.console import bold +from sphinx.util.fileutil import copy_asset from sphinx.util.pycompat import htmlescape -from sphinx.util.matching import compile_matchers +from sphinx.util.matching import Matcher from sphinx.errors import SphinxError import plistlib @@ -107,17 +107,15 @@ class AppleHelpBuilder(StandaloneHTMLBuilder): self.finish_tasks.add_task(self.build_helpbook) def copy_localized_files(self): - source_dir = path.join(self.confdir, - self.config.applehelp_locale + '.lproj') + source_dir = path.join(self.confdir, self.config.applehelp_locale + '.lproj') target_dir = self.outdir if path.isdir(source_dir): self.info(bold('copying localized files... '), nonl=True) - ctx = self.globalcontext.copy() - matchers = compile_matchers(self.config.exclude_patterns) - copy_static_entry(source_dir, target_dir, self, ctx, - exclude_matchers=matchers) + excluded = Matcher(self.config.exclude_patterns + ['**/.*']) + copy_asset(source_dir, target_dir, excluded, + context=self.globalcontext, renderer=self.templates) self.info('done') diff --git a/sphinx/builders/changes.py b/sphinx/builders/changes.py index 18f94adca..1bccb67d9 100644 --- a/sphinx/builders/changes.py +++ b/sphinx/builders/changes.py @@ -15,12 +15,12 @@ from os import path from six import iteritems from sphinx import package_dir -from sphinx.util import copy_static_entry from sphinx.locale import _ from sphinx.theming import Theme from sphinx.builders import Builder from sphinx.util.osutil import ensuredir, os_path from sphinx.util.console import bold +from sphinx.util.fileutil import copy_asset_file from sphinx.util.pycompat import htmlescape @@ -138,12 +138,10 @@ class ChangesBuilder(Builder): f.write(self.templates.render('changes/rstsource.html', ctx)) themectx = dict(('theme_' + key, val) for (key, val) in iteritems(self.theme.get_options({}))) - copy_static_entry(path.join(package_dir, 'themes', 'default', - 'static', 'default.css_t'), - self.outdir, self, themectx) - copy_static_entry(path.join(package_dir, 'themes', 'basic', - 'static', 'basic.css'), - self.outdir, self) + copy_asset_file(path.join(package_dir, 'themes', 'default', 'static', 'default.css_t'), + self.outdir, context=themectx, renderer=self.templates) + copy_asset_file(path.join(package_dir, 'themes', 'basic', 'static', 'basic.css'), + self.outdir) def hl(self, text, version): text = htmlescape(text) diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 9e8600968..f670c1399 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -27,13 +27,13 @@ from docutils.frontend import OptionParser from docutils.readers.doctree import Reader as DoctreeReader from sphinx import package_dir, __display_version__ -from sphinx.util import jsonimpl, copy_static_entry +from sphinx.util import jsonimpl from sphinx.util.i18n import format_date from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \ movefile, copyfile from sphinx.util.nodes import inline_all_toctrees from sphinx.util.fileutil import copy_asset -from sphinx.util.matching import patmatch, Matcher +from sphinx.util.matching import patmatch, Matcher, DOTFILES from sphinx.config import string_classes from sphinx.locale import _, l_ from sphinx.search import js_index @@ -614,11 +614,10 @@ class StandaloneHTMLBuilder(Builder): # then, copy over theme-supplied static files if self.theme: - themeentries = [path.join(themepath, 'static') - for themepath in self.theme.get_dirchain()[::-1]] - for entry in themeentries: - copy_static_entry(entry, path.join(self.outdir, '_static'), - self, ctx) + for theme_path in self.theme.get_dirchain()[::-1]: + entry = path.join(theme_path, 'static') + copy_asset(entry, path.join(self.outdir, '_static'), excluded=DOTFILES, + context=ctx, renderer=self.templates) # then, copy over all user-supplied static files excluded = Matcher(self.config.exclude_patterns + ["**/.*"]) for static_path in self.config.html_static_path: diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 036e57ac0..a0dababc8 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -149,7 +149,7 @@ class FilenameUniqDict(dict): def copy_static_entry(source, targetdir, builder, context={}, exclude_matchers=(), level=0): - """Copy a HTML builder static_path entry from source to targetdir. + """[DEPRECATED] Copy a HTML builder static_path entry from source to targetdir. Handles all possible cases of files, directories and subdirectories. """ diff --git a/sphinx/util/fileutil.py b/sphinx/util/fileutil.py index 8ab590472..1d2797b9c 100644 --- a/sphinx/util/fileutil.py +++ b/sphinx/util/fileutil.py @@ -26,6 +26,9 @@ def copy_asset_file(source, destination, context=None, renderer=None): :param context: The template variables. If not given, template files are simply copied :param renderer: The template engine """ + if not os.path.exists(source): + return + if os.path.exists(destination) and os.path.isdir(destination): # Use source filename if destination points a directory destination = os.path.join(destination, os.path.basename(source)) @@ -54,6 +57,9 @@ def copy_asset(source, destination, excluded=lambda path: False, context=None, r :param context: The template variables. If not given, template files are simply copied :param renderer: The template engine """ + if not os.path.exists(source): + return + ensuredir(destination) if os.path.isfile(source): copy_asset_file(source, destination, context, renderer) diff --git a/sphinx/util/matching.py b/sphinx/util/matching.py index 8f17980ca..fc7750be9 100644 --- a/sphinx/util/matching.py +++ b/sphinx/util/matching.py @@ -80,6 +80,9 @@ class Matcher(object): return any(pat(string) for pat in self.patterns) +DOTFILES = Matcher(['**/.*']) + + _pat_cache = {}