Merge pull request #6928 from tk0miya/deprecate_detect_encoding

Deprecate sphinx.util:detect_encoding()

CHANGES

@@ -10,10 +10,13 @@ Incompatible changes
 Deprecated
 ----------
 
+* The ``decode`` argument of ``sphinx.pycode.ModuleAnalyzer()``
 * ``sphinx.environment.BuildEnvironment.indexentries``
 * ``sphinx.environment.collectors.indexentries.IndexEntriesCollector``
 * ``sphinx.io.FiletypeNotFoundError``
 * ``sphinx.io.get_filetype()``
+* ``sphinx.pycode.ModuleAnalyzer.encoding``
+* ``sphinx.util.detect_encoding()``
 * ``sphinx.util.get_module_source()``
 
 Features added

doc/extdev/deprecated.rst

@@ -26,6 +26,11 @@ The following is a list of deprecated interfaces.
      - (will be) Removed
      - Alternatives
 
+   * - ``decode`` argument of ``sphinx.pycode.ModuleAnalyzer()``
+     - 2.4
+     - 4.0
+     - N/A
+
    * - ``sphinx.environment.BuildEnvironment.indexentries``
      - 2.4
      - 4.0
@@ -51,6 +56,16 @@ The following is a list of deprecated interfaces.
      - 4.0
      - N/A
 
+   * - ``sphinx.pycode.ModuleAnalyzer.encoding``
+     - 2.4
+     - 4.0
+     - N/A
+
+   * - ``sphinx.util.detect_encoding()``
+     - 2.4
+     - 4.0
+     - ``tokenize.detect_encoding()``
+
    * - ``sphinx.builders.gettext.POHEADER``
      - 2.3
      - 4.0

sphinx/pycode/__init__.py

@@ -9,15 +9,17 @@
 """
 
 import re
+import tokenize
+import warnings
 from importlib import import_module
 from io import StringIO
 from os import path
 from typing import Any, Dict, IO, List, Tuple, Optional
 from zipfile import ZipFile
 
+from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.errors import PycodeError
 from sphinx.pycode.parser import Parser
-from sphinx.util import detect_encoding
 
 
 class ModuleAnalyzer:
@@ -82,8 +84,8 @@ class ModuleAnalyzer:
         if ('file', filename) in cls.cache:
             return cls.cache['file', filename]
         try:
-            with open(filename, 'rb') as f:
-                obj = cls(f, modname, filename)
+            with tokenize.open(filename) as f:
+                obj = cls(f, modname, filename, decoded=True)
                 cls.cache['file', filename] = obj
         except Exception as err:
             if '.egg' + path.sep in filename:
@@ -130,11 +132,13 @@ class ModuleAnalyzer:
         # cache the source code as well
         pos = source.tell()
         if not decoded:
-            self.encoding = detect_encoding(source.readline)
+            warnings.warn('decode option for ModuleAnalyzer is deprecated.',
+                          RemovedInSphinx40Warning)
+            self._encoding, _ = tokenize.detect_encoding(source.readline)
             source.seek(pos)
-            self.code = source.read().decode(self.encoding)
+            self.code = source.read().decode(self._encoding)
         else:
-            self.encoding = None
+            self._encoding = None
             self.code = source.read()
 
         # will be filled by parse()
@@ -145,7 +149,7 @@ class ModuleAnalyzer:
     def parse(self) -> None:
         """Parse the source code."""
         try:
-            parser = Parser(self.code, self.encoding)
+            parser = Parser(self.code, self._encoding)
             parser.parse()
 
             self.attr_docs = {}
@@ -173,3 +177,9 @@ class ModuleAnalyzer:
             self.parse()
 
         return self.tags
+
+    @property
+    def encoding(self) -> str:
+        warnings.warn('ModuleAnalyzer.encoding is deprecated.',
+                      RemovedInSphinx40Warning)
+        return self._encoding

sphinx/util/__init__.py

@@ -346,6 +346,8 @@ _coding_re = re.compile(r'coding[:=]\s*([-\w.]+)')
 
 def detect_encoding(readline: Callable[[], bytes]) -> str:
     """Like tokenize.detect_encoding() from Py3k, but a bit simplified."""
+    warnings.warn('sphinx.util.detect_encoding() is deprecated',
+                  RemovedInSphinx40Warning)
 
     def read_or_stop() -> bytes:
         try:

tests/test_pycode.py

@@ -32,14 +32,12 @@ def test_ModuleAnalyzer_for_string():
     analyzer = ModuleAnalyzer.for_string('print("Hello world")', 'module_name')
     assert analyzer.modname == 'module_name'
     assert analyzer.srcname == '<string>'
-    assert analyzer.encoding is None
 
 
 def test_ModuleAnalyzer_for_file():
     analyzer = ModuleAnalyzer.for_string(SPHINX_MODULE_PATH, 'sphinx')
     assert analyzer.modname == 'sphinx'
     assert analyzer.srcname == '<string>'
-    assert analyzer.encoding is None
 
 
 def test_ModuleAnalyzer_for_module(rootdir):
@@ -47,8 +45,7 @@ def test_ModuleAnalyzer_for_module(rootdir):
     assert analyzer.modname == 'sphinx'
     assert analyzer.srcname in (SPHINX_MODULE_PATH,
                                 os.path.abspath(SPHINX_MODULE_PATH))
-    # source should be loaded via native loader, so don`t know file enconding
-    assert analyzer.encoding == None
+
     path = rootdir / 'test-pycode'
     sys.path.insert(0, path)
     try:
@@ -57,7 +54,6 @@ def test_ModuleAnalyzer_for_module(rootdir):
         assert docs == {('', 'X'): ['It MUST look like X="\u0425"', '']}
     finally:
         sys.path.pop(0)
-    
 
 
 def test_ModuleAnalyzer_for_file_in_egg(rootdir):