Add ImageDownloader transform to support remote images on some builders

sphinx/application.py

@@ -92,6 +92,7 @@ builtin_extensions = (
     'sphinx.directives.patches',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
+    'sphinx.transforms.post_transforms.images',
     # collectors should be loaded by specific order
     'sphinx.environment.collectors.dependencies',
     'sphinx.environment.collectors.asset',

sphinx/builders/__init__.py

@@ -61,6 +61,11 @@ class Builder(object):
     # support translation
     use_message_catalog = True
 
+    #: The list of MIME types of image formats supported by the builder.
+    #: Image files are searched in the order in which they appear here.
+    supported_image_types = []  # type: List[unicode]
+    supported_remote_images = True
+
     def __init__(self, app):
         # type: (Sphinx) -> None
         self.srcdir = app.srcdir
@@ -157,10 +162,6 @@ class Builder(object):
         """Return list of paths for assets (ex. templates, CSS, etc.)."""
         return []
 
-    #: The list of MIME types of image formats supported by the builder.
-    #: Image files are searched in the order in which they appear here.
-    supported_image_types = []  # type: List[unicode]
-
     def post_process_images(self, doctree):
         # type: (nodes.Node) -> None
         """Pick the best candidate for all image URIs."""

sphinx/builders/latex.py

@@ -51,6 +51,7 @@ class LaTeXBuilder(Builder):
     name = 'latex'
     format = 'latex'
     supported_image_types = ['application/pdf', 'image/png', 'image/jpeg']
+    supported_remote_images = False
 
     def init(self):
         # type: () -> None

sphinx/transforms/post_transforms/images.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.transforms.post_transforms.images
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Docutils transforms used by Sphinx.
+
+    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+
+from docutils import nodes
+
+from sphinx.transforms import SphinxTransform
+from sphinx.util import logging, requests
+from sphinx.util.osutil import ensuredir
+
+if False:
+    # For type annotation
+    from typing import Any, Dict  # NOQA
+    from sphinx.application import Sphinx  # NOQA
+
+
+logger = logging.getLogger(__name__)
+
+
+class BaseImageConverter(SphinxTransform):
+    def apply(self):
+        # type: () -> None
+        for node in self.document.traverse(nodes.image):
+            if self.match(node):
+                self.handle(node)
+
+    def match(self, node):
+        # type: (nodes.Node) -> bool
+        return True
+
+    def handle(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+
+class ImageDownloader(BaseImageConverter):
+    default_priority = 100
+
+    def match(self, node):
+        # type: (nodes.Node) -> bool
+        if self.app.builder.supported_remote_images:
+            return False
+        else:
+            return '://' in node['uri']
+
+    def handle(self, node):
+        # type: (nodes.Node) -> None
+        imgdir = os.path.join(self.app.doctreedir, 'images')
+        basename = os.path.basename(node['uri'])
+        if '?' in basename:
+            basename = basename.split('?')[0]
+        dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/",
+                                                             ord("&"): u"/"})
+        ensuredir(os.path.join(imgdir, dirname))
+        path = os.path.join(imgdir, dirname, basename)
+        with open(path, 'wb') as f:
+            r = requests.get(node['uri'])
+            f.write(r.content)
+
+        node['candidates'].pop('?')
+        node['candidates']['*'] = path
+        node['uri'] = path
+        self.app.env.images.add_file(self.env.docname, path)
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.add_post_transform(ImageDownloader)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

tests/root/images.txt

@@ -16,7 +16,7 @@ Sphinx image handling
 .. image:: img.*
 
 .. a non-local image URI
-.. image:: http://www.python.org/logo.png
+.. image:: https://www.python.org/static/img/python-logo.png
 
 .. an image with subdir and unspecified extension
 .. image:: subdir/simg.*

tests/roots/test-images/index.rst

@@ -14,3 +14,6 @@ test-image
    The caption of img
 
 .. image:: testimäge.png
+
+.. a remote image
+.. image:: https://www.python.org/static/img/python-logo.png

tests/test_build_html.py

@@ -1224,3 +1224,13 @@ def test_html_raw_directive(app, status, warning):
 def test_alternate_stylesheets(app, cached_etree_parse, fname, expect):
     app.build()
     check_xpath(cached_etree_parse(app.outdir / fname), fname, *expect)
+
+
+@pytest.mark.sphinx('html', testroot='images')
+def test_html_remote_images(app, status, warning):
+    app.builder.build_all()
+
+    result = (app.outdir / 'index.html').text(encoding='utf8')
+    assert ('<img alt="https://www.python.org/static/img/python-logo.png" '
+            'src="https://www.python.org/static/img/python-logo.png" />' in result)
+    assert not (app.outdir / 'python-logo.png').exists()

tests/test_build_latex.py

@@ -1042,3 +1042,12 @@ def test_latex_raw_directive(app, status, warning):
     # with substitution
     assert 'HTML: abc  ghi' in result
     assert 'LaTeX: abc def ghi' in result
+
+
+@pytest.mark.sphinx('latex', testroot='images')
+def test_latex_remote_images(app, status, warning):
+    app.builder.build_all()
+
+    result = (app.outdir / 'Python.tex').text(encoding='utf8')
+    assert '\\sphinxincludegraphics{{python-logo}.png}' in result
+    assert (app.outdir / 'python-logo.png').exists()