Specify encoding

setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages, setup
 
 import sphinx
 
-with open('README.rst') as f:
+with open('README.rst', encoding='utf-8') as f:
     long_desc = f.read()
 
 if sys.version_info < (3, 6):

sphinx/builders/html/__init__.py

@@ -370,7 +370,7 @@ class StandaloneHTMLBuilder(Builder):
 
     def get_outdated_docs(self) -> Iterator[str]:
         try:
-            with open(path.join(self.outdir, '.buildinfo')) as fp:
+            with open(path.join(self.outdir, '.buildinfo'), encoding="utf-8") as fp:
                 buildinfo = BuildInfo.load(fp)
 
             if self.build_info != buildinfo:
@@ -763,11 +763,13 @@ class StandaloneHTMLBuilder(Builder):
 
     def create_pygments_style_file(self) -> None:
         """create a style file for pygments."""
-        with open(path.join(self.outdir, '_static', 'pygments.css'), 'w') as f:
+        with open(path.join(self.outdir, '_static', 'pygments.css'), 'w',
+                  encoding="utf-8") as f:
             f.write(self.highlighter.get_stylesheet())
 
         if self.dark_highlighter:
-            with open(path.join(self.outdir, '_static', 'pygments_dark.css'), 'w') as f:
+            with open(path.join(self.outdir, '_static', 'pygments_dark.css'), 'w',
+                      encoding="utf-8") as f:
                 f.write(self.dark_highlighter.get_stylesheet())
 
     def copy_translation_js(self) -> None:
@@ -853,7 +855,7 @@ class StandaloneHTMLBuilder(Builder):
 
     def write_buildinfo(self) -> None:
         try:
-            with open(path.join(self.outdir, '.buildinfo'), 'w') as fp:
+            with open(path.join(self.outdir, '.buildinfo'), 'w', encoding="utf-8") as fp:
                 self.build_info.dump(fp)
         except OSError as exc:
             logger.warning(__('Failed to write build info file: %r'), exc)

sphinx/builders/latex/__init__.py

@@ -241,7 +241,7 @@ class LaTeXBuilder(Builder):
     def write_stylesheet(self) -> None:
         highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style)
         stylesheet = path.join(self.outdir, 'sphinxhighlight.sty')
-        with open(stylesheet, 'w') as f:
+        with open(stylesheet, 'w', encoding="utf-8") as f:
             f.write('\\NeedsTeXFormat{LaTeX2e}[1995/12/01]\n')
             f.write('\\ProvidesPackage{sphinxhighlight}'
                     '[2016/05/29 stylesheet for highlighting with pygments]\n')

sphinx/builders/latex/theming.py

@@ -73,7 +73,7 @@ class UserTheme(Theme):
     def __init__(self, name: str, filename: str) -> None:
         super().__init__(name)
         self.config = configparser.RawConfigParser()
-        self.config.read(path.join(filename))
+        self.config.read(path.join(filename), encoding='utf-8')
 
         for key in self.REQUIRED_CONFIG_KEYS:
             try:

sphinx/builders/linkcheck.py

@@ -184,8 +184,10 @@ class CheckExternalLinksBuilder(DummyBuilder):
         checker = HyperlinkAvailabilityChecker(self.env, self.config)
         logger.info('')
 
-        with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
+        output_text = path.join(self.outdir, 'output.txt')
-             open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
+        output_json = path.join(self.outdir, 'output.json')
+        with open(output_text, 'w', encoding="utf-8") as self.txt_outfile,\
+             open(output_json, 'w', encoding="utf-8") as self.json_outfile:
             for result in checker.check(self.hyperlinks):
                 self.process_result(result)
 

sphinx/cmd/build.py

@@ -236,7 +236,7 @@ def build_main(argv: List[str] = sys.argv[1:]) -> int:
         try:
             warnfile = abspath(args.warnfile)
             ensuredir(path.dirname(warnfile))
-            warnfp = open(args.warnfile, 'w')
+            warnfp = open(args.warnfile, 'w', encoding="utf-8")
         except Exception as exc:
             parser.error(__('cannot open warning file %r: %s') % (
                 args.warnfile, exc))

sphinx/cmd/quickstart.py

@@ -367,7 +367,7 @@ def generate(d: Dict, overwrite: bool = True, silent: bool = False, templatedir:
     conf_path = os.path.join(templatedir, 'conf.py_t') if templatedir else None
     if not conf_path or not path.isfile(conf_path):
         conf_path = os.path.join(package_dir, 'templates', 'quickstart', 'conf.py_t')
-    with open(conf_path) as f:
+    with open(conf_path, encoding="utf-8") as f:
         conf_text = f.read()
 
     write_file(path.join(srcdir, 'conf.py'), template.render_string(conf_text, d))

sphinx/ext/coverage.py

@@ -90,7 +90,7 @@ class CoverageBuilder(Builder):
         c_objects = self.env.domaindata['c']['objects']
         for filename in self.c_sourcefiles:
             undoc: Set[Tuple[str, str]] = set()
-            with open(filename) as f:
+            with open(filename, encoding="utf-8") as f:
                 for line in f:
                     for key, regex in self.c_regexes:
                         match = regex.match(line)
@@ -108,7 +108,7 @@ class CoverageBuilder(Builder):
 
     def write_c_coverage(self) -> None:
         output_file = path.join(self.outdir, 'c.txt')
-        with open(output_file, 'w') as op:
+        with open(output_file, 'w', encoding="utf-8") as op:
             if self.config.coverage_write_headline:
                 write_header(op, 'Undocumented C API elements', '=')
             op.write('\n')
@@ -227,7 +227,7 @@ class CoverageBuilder(Builder):
     def write_py_coverage(self) -> None:
         output_file = path.join(self.outdir, 'python.txt')
         failed = []
-        with open(output_file, 'w') as op:
+        with open(output_file, 'w', encoding="utf-8") as op:
             if self.config.coverage_write_headline:
                 write_header(op, 'Undocumented Python objects', '=')
             keys = sorted(self.py_undoc.keys())

sphinx/ext/githubpages.py

@@ -11,13 +11,14 @@ from sphinx.environment import BuildEnvironment
 
 def create_nojekyll_and_cname(app: Sphinx, env: BuildEnvironment) -> None:
     if app.builder.format == 'html':
-        open(os.path.join(app.builder.outdir, '.nojekyll'), 'wt').close()
+        open(os.path.join(app.builder.outdir, '.nojekyll'), 'wb').close()
 
         html_baseurl = app.config.html_baseurl
         if html_baseurl:
             domain = urllib.parse.urlparse(html_baseurl).hostname
             if domain and not domain.endswith(".github.io"):
-                with open(os.path.join(app.builder.outdir, 'CNAME'), 'wt') as f:
+                with open(os.path.join(app.builder.outdir, 'CNAME'), 'w',
+                          encoding="utf-8") as f:
                     # NOTE: don't write a trailing newline. The `CNAME` file that's
                     # auto-generated by the Github UI doesn't have one.
                     f.write(domain)

sphinx/ext/imgmath.py

@@ -56,7 +56,7 @@ depthsvgcomment_re = re.compile(r'<!-- DEPTH=(-?\d+) -->')
 def read_svg_depth(filename: str) -> int:
     """Read the depth from comment at last line of SVG file
     """
-    with open(filename) as f:
+    with open(filename, encoding="utf-8") as f:
         for line in f:  # noqa: B007
             pass
         # Only last line is checked
@@ -69,7 +69,7 @@ def read_svg_depth(filename: str) -> int:
 def write_svg_depth(filename: str, depth: int) -> None:
     """Write the depth to SVG file as a comment at end of file
     """
-    with open(filename, 'a') as f:
+    with open(filename, 'a', encoding="utf-8") as f:
         f.write('\n<!-- DEPTH=%s -->' % depth)
 
 

sphinx/search/__init__.py

@@ -439,9 +439,9 @@ class IndexBuilder:
         """Returns JS code that will be inserted into language_data.js."""
         if self.lang.js_stemmer_rawcode:
             js_dir = path.join(package_dir, 'search', 'minified-js')
-            with open(path.join(js_dir, 'base-stemmer.js')) as js_file:
+            with open(path.join(js_dir, 'base-stemmer.js'), encoding='utf-8') as js_file:
                 base_js = js_file.read()
-            with open(path.join(js_dir, self.lang.js_stemmer_rawcode)) as js_file:
+            with open(path.join(js_dir, self.lang.js_stemmer_rawcode), encoding='utf-8') as js_file:
                 language_js = js_file.read()
             return ('%s\n%s\nStemmer = %sStemmer;' %
                     (base_js, language_js, self.lang.language_name))

sphinx/theming.py

@@ -64,7 +64,7 @@ class Theme:
             extract_zip(theme_path, self.themedir)
 
         self.config = configparser.RawConfigParser()
-        self.config.read(path.join(self.themedir, THEMECONF))
+        self.config.read(path.join(self.themedir, THEMECONF), encoding='utf-8')
 
         try:
             inherit = self.config.get('theme', 'inherit')

sphinx/util/docutils.py

@@ -338,6 +338,7 @@ class SphinxFileOutput(FileOutput):
 
     def __init__(self, **kwargs: Any) -> None:
         self.overwrite_if_changed = kwargs.pop('overwrite_if_changed', False)
+        kwargs.setdefault('encoding', 'utf-8')
         super().__init__(**kwargs)
 
     def write(self, data: str) -> str:

tests/roots/test-apidoc-toc/mypackage/main.py

@@ -10,6 +10,6 @@ if __name__ == "__main__":
 
     res_path = \
         os.path.join(os.path.dirname(mod_resource.__file__), 'resource.txt')
-    with open(res_path) as f:
+    with open(res_path, encoding='utf-8') as f:
         text = f.read()
     print("From mod_resource:resource.txt -> {}".format(text))

tests/test_build_linkcheck.py

@@ -289,7 +289,7 @@ def test_linkcheck_allowed_redirects(app, warning):
     with http_server(make_redirect_handler(support_head=False)):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         records = [json.loads(l) for l in fp.readlines()]
 
     assert len(records) == 2
@@ -318,7 +318,7 @@ def test_invalid_ssl(app):
     with http_server(OKHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content["status"] == "broken"
     assert content["filename"] == "index.rst"
@@ -332,7 +332,7 @@ def test_connect_to_selfsigned_fails(app):
     with https_server(OKHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content["status"] == "broken"
     assert content["filename"] == "index.rst"
@@ -347,7 +347,7 @@ def test_connect_to_selfsigned_with_tls_verify_false(app):
     with https_server(OKHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content == {
         "code": 0,
@@ -365,7 +365,7 @@ def test_connect_to_selfsigned_with_tls_cacerts(app):
     with https_server(OKHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content == {
         "code": 0,
@@ -383,7 +383,7 @@ def test_connect_to_selfsigned_with_requests_env_var(monkeypatch, app):
     with https_server(OKHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content == {
         "code": 0,
@@ -401,7 +401,7 @@ def test_connect_to_selfsigned_nonexistent_cert_file(app):
     with https_server(OKHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content == {
         "code": 0,
@@ -429,7 +429,7 @@ def test_TooManyRedirects_on_HEAD(app):
     with http_server(InfiniteRedirectOnHeadHandler):
         app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = json.load(fp)
     assert content == {
         "code": 0,
@@ -623,7 +623,7 @@ def test_get_after_head_raises_connection_error(app):
 def test_linkcheck_exclude_documents(app):
     app.build()
 
-    with open(app.outdir / 'output.json') as fp:
+    with open(app.outdir / 'output.json', encoding='utf-8') as fp:
         content = [json.loads(record) for record in fp]
 
     assert content == [

tests/test_ext_apidoc.py

@@ -61,16 +61,16 @@ def test_pep_0420_enabled(make_app, apidoc):
     assert (outdir / 'a.b.e.rst').isfile()
     assert (outdir / 'a.b.x.rst').isfile()
 
-    with open(outdir / 'a.b.c.rst') as f:
+    with open(outdir / 'a.b.c.rst', encoding='utf-8') as f:
         rst = f.read()
         assert "automodule:: a.b.c.d\n" in rst
         assert "automodule:: a.b.c\n" in rst
 
-    with open(outdir / 'a.b.e.rst') as f:
+    with open(outdir / 'a.b.e.rst', encoding='utf-8') as f:
         rst = f.read()
         assert "automodule:: a.b.e.f\n" in rst
 
-    with open(outdir / 'a.b.x.rst') as f:
+    with open(outdir / 'a.b.x.rst', encoding='utf-8') as f:
         rst = f.read()
         assert "automodule:: a.b.x.y\n" in rst
         assert "automodule:: a.b.x\n" not in rst
@@ -85,15 +85,15 @@ def test_pep_0420_enabled(make_app, apidoc):
     assert (builddir / 'a.b.e.txt').isfile()
     assert (builddir / 'a.b.x.txt').isfile()
 
-    with open(builddir / 'a.b.c.txt') as f:
+    with open(builddir / 'a.b.c.txt', encoding='utf-8') as f:
         txt = f.read()
         assert "a.b.c package\n" in txt
 
-    with open(builddir / 'a.b.e.txt') as f:
+    with open(builddir / 'a.b.e.txt', encoding='utf-8') as f:
         txt = f.read()
         assert "a.b.e.f module\n" in txt
 
-    with open(builddir / 'a.b.x.txt') as f:
+    with open(builddir / 'a.b.x.txt', encoding='utf-8') as f:
         txt = f.read()
         assert "a.b.x namespace\n" in txt
 
@@ -111,15 +111,15 @@ def test_pep_0420_enabled_separate(make_app, apidoc):
     assert (outdir / 'a.b.x.rst').isfile()
     assert (outdir / 'a.b.x.y.rst').isfile()
 
-    with open(outdir / 'a.b.c.rst') as f:
+    with open(outdir / 'a.b.c.rst', encoding='utf-8') as f:
         rst = f.read()
         assert ".. toctree::\n   :maxdepth: 4\n\n   a.b.c.d\n" in rst
 
-    with open(outdir / 'a.b.e.rst') as f:
+    with open(outdir / 'a.b.e.rst', encoding='utf-8') as f:
         rst = f.read()
         assert ".. toctree::\n   :maxdepth: 4\n\n   a.b.e.f\n" in rst
 
-    with open(outdir / 'a.b.x.rst') as f:
+    with open(outdir / 'a.b.x.rst', encoding='utf-8') as f:
         rst = f.read()
         assert ".. toctree::\n   :maxdepth: 4\n\n   a.b.x.y\n" in rst
 
@@ -135,15 +135,15 @@ def test_pep_0420_enabled_separate(make_app, apidoc):
     assert (builddir / 'a.b.x.txt').isfile()
     assert (builddir / 'a.b.x.y.txt').isfile()
 
-    with open(builddir / 'a.b.c.txt') as f:
+    with open(builddir / 'a.b.c.txt', encoding='utf-8') as f:
         txt = f.read()
         assert "a.b.c package\n" in txt
 
-    with open(builddir / 'a.b.e.f.txt') as f:
+    with open(builddir / 'a.b.e.f.txt', encoding='utf-8') as f:
         txt = f.read()
         assert "a.b.e.f module\n" in txt
 
-    with open(builddir / 'a.b.x.txt') as f:
+    with open(builddir / 'a.b.x.txt', encoding='utf-8') as f:
         txt = f.read()
         assert "a.b.x namespace\n" in txt
 
@@ -169,7 +169,7 @@ def test_pep_0420_disabled_top_level_verify(make_app, apidoc):
     assert (outdir / 'c.rst').isfile()
     assert not (outdir / 'x.rst').exists()
 
-    with open(outdir / 'c.rst') as f:
+    with open(outdir / 'c.rst', encoding='utf-8') as f:
         rst = f.read()
         assert "c package\n" in rst
         assert "automodule:: c.d\n" in rst
@@ -194,7 +194,7 @@ def test_trailing_underscore(make_app, apidoc):
     print(app._warning.getvalue())
 
     builddir = outdir / '_build' / 'text'
-    with open(builddir / 'package_.txt') as f:
+    with open(builddir / 'package_.txt', encoding='utf-8') as f:
         rst = f.read()
         assert "package_ package\n" in rst
         assert "package_.module_ module\n" in rst
@@ -295,7 +295,7 @@ def test_extension_parsed(make_app, apidoc):
     outdir = apidoc.outdir
     assert (outdir / 'conf.py').isfile()
 
-    with open(outdir / 'conf.py') as f:
+    with open(outdir / 'conf.py', encoding='utf-8') as f:
         rst = f.read()
         assert "sphinx.ext.mathjax" in rst
 
@@ -363,7 +363,7 @@ def test_toc_all_references_should_exist_pep420_disabled(make_app, apidoc):
 
 def extract_toc(path):
     """Helper: Extract toc section from package rst file"""
-    with open(path) as f:
+    with open(path, encoding='utf-8') as f:
         rst = f.read()
 
     # Read out the part containing the toctree
@@ -389,12 +389,12 @@ def test_subpackage_in_toc(make_app, apidoc):
     assert (outdir / 'conf.py').isfile()
 
     assert (outdir / 'parent.rst').isfile()
-    with open(outdir / 'parent.rst') as f:
+    with open(outdir / 'parent.rst', encoding='utf-8') as f:
         parent = f.read()
     assert 'parent.child' in parent
 
     assert (outdir / 'parent.child.rst').isfile()
-    with open(outdir / 'parent.child.rst') as f:
+    with open(outdir / 'parent.child.rst', encoding='utf-8') as f:
         parent_child = f.read()
     assert 'parent.child.foo' in parent_child
 

tests/test_intl.py

@@ -29,7 +29,7 @@ pygments_version = tuple(int(v) for v in pygments.__version__.split('.'))
 
 
 def read_po(pathname):
-    with pathname.open() as f:
+    with pathname.open(encoding='utf-8') as f:
         return pofile.read_po(f)
 
 

tests/test_smartquotes.py

@@ -18,7 +18,7 @@ def test_basic(app, status, warning):
 def test_literals(app, status, warning):
     app.build()
 
-    with (app.outdir / 'literals.html').open() as html_file:
+    with (app.outdir / 'literals.html').open(encoding='utf-8') as html_file:
         etree = HTMLParser(namespaceHTMLElements=False).parse(html_file)
 
     for code_element in etree.iter('code'):

utils/bump_version.py

@@ -27,7 +27,7 @@ def bump_version(path, version_info, in_develop=True):
     if in_develop:
         version += '+'
 
-    with open(path, 'r+') as f:
+    with open(path, 'r+', encoding='utf-8') as f:
         body = f.read()
         body = re.sub(r"(?<=__version__ = ')[^']+", version, body)
         body = re.sub(r"(?<=__released__ = ')[^']+", release, body)
@@ -88,7 +88,7 @@ class Changes:
         self.fetch_version()
 
     def fetch_version(self):
-        with open(self.path) as f:
+        with open(self.path, encoding='utf-8') as f:
             version = f.readline().strip()
             matched = re.search(r'^Release (.*) \((.*)\)$', version)
             if matched is None:
@@ -105,7 +105,7 @@ class Changes:
         release_date = datetime.now().strftime('%b %d, %Y')
         heading = 'Release %s (released %s)' % (self.version, release_date)
 
-        with open(self.path, 'r+') as f:
+        with open(self.path, 'r+', encoding='utf-8') as f:
             f.readline()  # skip first two lines
             f.readline()
             body = f.read()
@@ -126,12 +126,12 @@ class Changes:
                                    version_info[4] or '')
         heading = 'Release %s (in development)' % version
 
-        with open(os.path.join(script_dir, 'CHANGES_template')) as f:
+        with open(os.path.join(script_dir, 'CHANGES_template'), encoding='utf-8') as f:
             f.readline()  # skip first two lines
             f.readline()
             tmpl = f.read()
 
-        with open(self.path, 'r+') as f:
+        with open(self.path, 'r+', encoding='utf-8') as f:
             body = f.read()
 
             f.seek(0)

utils/doclinter.py

@@ -12,7 +12,7 @@ LEADING_SPACES = re.compile(r'^(\s*)')
 
 
 def lint(path: str) -> int:
-    with open(path) as f:
+    with open(path, encoding='utf-8') as f:
         document = f.readlines()
 
     errors = 0