intersphinx: Use requests package

sphinx/ext/intersphinx.py

@@ -34,7 +34,6 @@ from os import path
 import re
 
 from six import iteritems, string_types
-from six.moves.urllib import request
 from six.moves.urllib.parse import urlsplit, urlunsplit
 from docutils import nodes
 from docutils.utils import relative_path
@@ -42,17 +41,9 @@ from docutils.utils import relative_path
 import sphinx
 from sphinx.locale import _
 from sphinx.builders.html import INVENTORY_FILENAME
+from sphinx.util.requests import requests
 
 
-default_handlers = [request.ProxyHandler(), request.HTTPRedirectHandler(),
-                    request.HTTPHandler()]
-try:
-    default_handlers.append(request.HTTPSHandler)
-except AttributeError:
-    pass
-
-default_opener = request.build_opener(*default_handlers)
-
 UTF8StreamReader = codecs.lookup('utf-8')[2]
 
 
@@ -183,17 +174,10 @@ def _read_from_url(url):
     :return: data read from resource described by *url*
     :rtype: ``file``-like object
     """
-    url, username, password = _strip_basic_auth(url)
-    if username is not None and password is not None:
-        # case: url contains basic auth creds
-        password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
-        password_mgr.add_password(None, url, username, password)
-        handler = request.HTTPBasicAuthHandler(password_mgr)
-        opener = request.build_opener(*(default_handlers + [handler]))
-    else:
-        opener = default_opener
-
-    return opener.open(url)
+    r = requests.get(url, stream=True)
+    r.raise_for_status()
+    r.raw.url = r.url
+    return r.raw
 
 
 def _get_safe_url(url):
@@ -239,8 +223,8 @@ def fetch_inventory(app, uri, inv):
                  '%s: %s' % (inv, err.__class__, err))
         return
     try:
-        if hasattr(f, 'geturl'):
-            newinv = f.geturl()
+        if hasattr(f, 'url'):
+            newinv = f.url
             if inv != newinv:
                 app.info('intersphinx inventory has moved: %s -> %s' % (inv, newinv))
 

tests/test_ext_intersphinx.py

@@ -81,47 +81,47 @@ def test_read_inventory_v2():
 
 
 @with_app()
-@mock.patch('sphinx.ext.intersphinx.read_inventory_v2')
+@mock.patch('sphinx.ext.intersphinx.read_inventory')
 @mock.patch('sphinx.ext.intersphinx._read_from_url')
-def test_fetch_inventory_redirection(app, status, warning, _read_from_url, read_inventory_v2):
+def test_fetch_inventory_redirection(app, status, warning, _read_from_url, read_inventory):
     _read_from_url().readline.return_value = '# Sphinx inventory version 2'.encode('utf-8')
 
     # same uri and inv, not redirected
-    _read_from_url().geturl.return_value = 'http://hostname/' + INVENTORY_FILENAME
+    _read_from_url().url = 'http://hostname/' + INVENTORY_FILENAME
     fetch_inventory(app, 'http://hostname/', 'http://hostname/' + INVENTORY_FILENAME)
     assert 'intersphinx inventory has moved' not in status.getvalue()
-    assert read_inventory_v2.call_args[0][1] == 'http://hostname/'
+    assert read_inventory.call_args[0][1] == 'http://hostname/'
 
     # same uri and inv, redirected
     status.seek(0)
     status.truncate(0)
-    _read_from_url().geturl.return_value = 'http://hostname/new/' + INVENTORY_FILENAME
+    _read_from_url().url = 'http://hostname/new/' + INVENTORY_FILENAME
 
     fetch_inventory(app, 'http://hostname/', 'http://hostname/' + INVENTORY_FILENAME)
     assert status.getvalue() == ('intersphinx inventory has moved: '
                                  'http://hostname/%s -> http://hostname/new/%s\n' %
                                  (INVENTORY_FILENAME, INVENTORY_FILENAME))
-    assert read_inventory_v2.call_args[0][1] == 'http://hostname/new'
+    assert read_inventory.call_args[0][1] == 'http://hostname/new'
 
     # different uri and inv, not redirected
     status.seek(0)
     status.truncate(0)
-    _read_from_url().geturl.return_value = 'http://hostname/new/' + INVENTORY_FILENAME
+    _read_from_url().url = 'http://hostname/new/' + INVENTORY_FILENAME
 
     fetch_inventory(app, 'http://hostname/', 'http://hostname/new/' + INVENTORY_FILENAME)
     assert 'intersphinx inventory has moved' not in status.getvalue()
-    assert read_inventory_v2.call_args[0][1] == 'http://hostname/'
+    assert read_inventory.call_args[0][1] == 'http://hostname/'
 
     # different uri and inv, redirected
     status.seek(0)
     status.truncate(0)
-    _read_from_url().geturl.return_value = 'http://hostname/other/' + INVENTORY_FILENAME
+    _read_from_url().url = 'http://hostname/other/' + INVENTORY_FILENAME
 
     fetch_inventory(app, 'http://hostname/', 'http://hostname/new/' + INVENTORY_FILENAME)
     assert status.getvalue() == ('intersphinx inventory has moved: '
                                  'http://hostname/new/%s -> http://hostname/other/%s\n' %
                                  (INVENTORY_FILENAME, INVENTORY_FILENAME))
-    assert read_inventory_v2.call_args[0][1] == 'http://hostname/'
+    assert read_inventory.call_args[0][1] == 'http://hostname/'
 
 
 @with_app()
@@ -255,41 +255,6 @@ class TestStripBasicAuth(unittest.TestCase):
         self.assertEqual('12345', actual_password)
 
 
-@mock.patch('six.moves.urllib.request.HTTPBasicAuthHandler')
-@mock.patch('six.moves.urllib.request.HTTPPasswordMgrWithDefaultRealm')
-@mock.patch('six.moves.urllib.request.build_opener')
-def test_readfromurl_authed(m_build_opener, m_HTTPPasswordMgrWithDefaultRealm,
-                            m_HTTPBasicAuthHandler):
-    # read from URL containing basic auth creds
-    password_mgr = mock.Mock()
-    m_HTTPPasswordMgrWithDefaultRealm.return_value = password_mgr
-
-    url = 'https://user:12345@domain.com/project/objects.inv'
-    _read_from_url(url)
-
-    m_HTTPPasswordMgrWithDefaultRealm.assert_called_once_with()
-    password_mgr.add_password.assert_called_with(
-        None, 'https://domain.com/project/objects.inv', 'user', '12345')
-
-
-@mock.patch('six.moves.urllib.request.HTTPBasicAuthHandler')
-@mock.patch('six.moves.urllib.request.HTTPPasswordMgrWithDefaultRealm')
-@mock.patch('sphinx.ext.intersphinx.default_opener')
-def test_readfromurl_unauthed(m_default_opener, m_HTTPPasswordMgrWithDefaultRealm,
-                              m_HTTPBasicAuthHandler):
-    # read from URL without auth creds
-    password_mgr = mock.Mock()
-    m_HTTPPasswordMgrWithDefaultRealm.return_value = password_mgr
-
-    url = 'https://domain.com/project/objects.inv'
-    _read_from_url(url)
-
-    # assert password manager not created
-    assert m_HTTPPasswordMgrWithDefaultRealm.call_args is None
-    # assert no password added to the password manager
-    assert password_mgr.add_password.call_args is None
-
-
 def test_getsafeurl_authed():
     """_get_safe_url() with a url with basic auth"""
     url = 'https://user:12345@domain.com/project/objects.inv'