Merge pull request #9309 from jamathews/4.0.x

Handle ConnectionError on HTTP HEAD Requests

CHANGES

@@ -67,6 +67,8 @@ Bugs fixed
 * #9270: html theme : pyramid theme generates incorrect logo links
 * #9217: manpage: The name of manpage directory that is generated by
   :confval:`man_make_section_directory` is not correct
+* #9306: Linkcheck reports broken link when remote server closes the connection
+  on HEAD request
 * #9280: py domain: "exceptions" module is not displayed
 * #9224: ``:param:`` and ``:type:`` fields does not support a type containing
   whitespace (ex. ``Dict[str, str]``)

sphinx/builders/linkcheck.py

@@ -26,7 +26,7 @@ from urllib.parse import unquote, urlparse, urlunparse
 from docutils import nodes
 from docutils.nodes import Element
 from requests import Response
-from requests.exceptions import HTTPError, TooManyRedirects
+from requests.exceptions import ConnectionError, HTTPError, TooManyRedirects
 
 from sphinx.application import Sphinx
 from sphinx.builders.dummy import DummyBuilder
@@ -456,7 +456,9 @@ class HyperlinkAvailabilityCheckWorker(Thread):
                                                  config=self.config, auth=auth_info,
                                                  **kwargs)
                         response.raise_for_status()
-                    except (HTTPError, TooManyRedirects) as err:
+                    # Servers drop the connection on HEAD requests, causing
+                    # ConnectionError.
+                    except (ConnectionError, HTTPError, TooManyRedirects) as err:
                         if isinstance(err, HTTPError) and err.response.status_code == 429:
                             raise
                         # retry with GET request if that fails, some servers

tests/test_build_linkcheck.py

@@ -579,3 +579,29 @@ def test_limit_rate_bails_out_after_waiting_max_time(app):
                                               rate_limits)
     next_check = worker.limit_rate(FakeResponse())
     assert next_check is None
+
+
+class ConnectionResetHandler(http.server.BaseHTTPRequestHandler):
+    def do_HEAD(self):
+        self.connection.close()
+
+    def do_GET(self):
+        self.send_response(200, "OK")
+        self.end_headers()
+
+
+@pytest.mark.sphinx('linkcheck', testroot='linkcheck-localserver', freshenv=True)
+def test_get_after_head_raises_connection_error(app):
+    with http_server(ConnectionResetHandler):
+        app.build()
+    content = (app.outdir / 'output.txt').read_text()
+    assert not content
+    content = (app.outdir / 'output.json').read_text()
+    assert json.loads(content) == {
+        "filename": "index.rst",
+        "lineno": 1,
+        "status": "working",
+        "code": 0,
+        "uri": "http://localhost:7777/",
+        "info": "",
+    }