From f190de75cdcee9a3b88673a615b56bb72527b0c7 Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Thu, 28 Aug 2014 22:41:52 +0900 Subject: [PATCH] Fix: pgen2 tokenizer doesn't recognize `...` literal (Ellipsis for py3). Closes #1547 I think pgen2 code derived from lib2to3 package. Basically, the package only support python2 code then it doesn't recognize `...` literal. --- CHANGES | 1 + sphinx/pycode/pgen2/tokenize.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 2f905c058..92837b2c7 100644 --- a/CHANGES +++ b/CHANGES @@ -215,6 +215,7 @@ Bugs fixed :rst:dir:`c:function`. Thanks to Takeshi Komiya. * PR#278: Fix section entries were shown twice if toctree has been put under only directive. Thanks to Takeshi Komiya. +* #1547: pgen2 tokenizer doesn't recognize `...` literal (Ellipsis for py3). Documentation ------------- diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py index f516f78ba..d62535050 100644 --- a/sphinx/pycode/pgen2/tokenize.py +++ b/sphinx/pycode/pgen2/tokenize.py @@ -33,6 +33,7 @@ __credits__ = \ 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' import string, re +from six import PY3 from sphinx.pycode.pgen2.token import * from sphinx.pycode.pgen2 import token @@ -84,6 +85,9 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", Bracket = '[][(){}]' Special = group(r'\r?\n', r'[:;.,`@]') +if PY3: + Ellipsis_ = r'\.{3}' + Special = group(Ellipsis_, Special) Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) @@ -356,8 +360,9 @@ def generate_tokens(readline): spos, epos, pos = (lnum, start), (lnum, end), end token, initial = line[start:end], line[start] - if initial in numchars or \ - (initial == '.' and token != '.'): # ordinary number + if initial in numchars or ( + initial == '.' and token not in ('.', '...') + ): # ordinary number yield (NUMBER, token, spos, epos, line) elif initial in '\r\n': newline = NEWLINE @@ -393,6 +398,8 @@ def generate_tokens(readline): yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name yield (NAME, token, spos, epos, line) + elif token in ('...',): # ordinary name + yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: yield (NL, token, spos, (lnum, pos), line)