Merge pull request #5068 from tk0miya/5019_autodoc_crashed_by_FF

Fix #5019: autodoc: crashed by Form Feed Character

CHANGES

@@ -26,6 +26,7 @@ Bugs fixed
 * C++, fix lookup of full template specializations with no template arguments.
 * #4667: C++, fix assertion on missing references in global scope when using
   intersphinx. Thanks to Alan M. Carroll.
+* #5019: autodoc: crashed by Form Feed Character
 
 Testing
 --------

sphinx/pycode/parser.py

@@ -34,6 +34,11 @@ else:
     ASSIGN_NODES = (ast.Assign)
 
 
+def filter_whitespace(code):
+    # type: (unicode) -> unicode
+    return code.replace('\f', ' ')  # replace FF (form feed) with whitespace
+
+
 def get_assign_targets(node):
     # type: (ast.AST) -> List[ast.expr]
     """Get list of targets from Assign and AnnAssign node."""
@@ -466,7 +471,7 @@ class Parser(object):
 
     def __init__(self, code, encoding='utf-8'):
         # type: (unicode, unicode) -> None
-        self.code = code
+        self.code = filter_whitespace(code)
         self.encoding = encoding
         self.comments = {}          # type: Dict[Tuple[unicode, unicode], unicode]
         self.deforders = {}         # type: Dict[unicode, int]

tests/test_pycode_parser.py

@@ -315,3 +315,12 @@ def test_decorators():
                                   'func3': ('def', 7, 9),
                                   'Foo': ('class', 11, 15),
                                   'Foo.method': ('def', 13, 15)}
+
+
+def test_formfeed_char():
+    source = ('class Foo:\n'
+              '\f\n'
+              '    attr = 1234  #: comment\n')
+    parser = Parser(source)
+    parser.parse()
+    assert parser.comments == {('Foo', 'attr'): 'comment'}