Merge pull request #5226 from tk0miya/5140_Accept_header_for_linkcheck

Add default Accept header to the linkcheck

CHANGES

@@ -182,6 +182,7 @@ Features added
 * #2820: autoclass directive supports nested class
 * Add ``app.add_html_math_renderer()`` to register a math renderer for HTML
 * Apply :confval:`trim_doctest_flags` to all builders (cf. text, manpages)
+* #5140: linkcheck: Add better Accept header to HTTP client
 
 Bugs fixed
 ----------

sphinx/builders/linkcheck.py

@@ -120,12 +120,15 @@ class CheckExternalLinksBuilder(Builder):
 
     def check_thread(self):
         # type: () -> None
-        kwargs = {}
+        kwargs = {
+            'allow_redirects': True,
+            'headers': {
+                'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8'
+            },
+        }
         if self.app.config.linkcheck_timeout:
             kwargs['timeout'] = self.app.config.linkcheck_timeout
 
-        kwargs['allow_redirects'] = True
-
         def check_uri():
             # type: () -> Tuple[unicode, unicode, int]
             # split off anchor