From 1172d1dba26f29c599eac8506d8c0e3fb170b4ee Mon Sep 17 00:00:00 2001 From: Alberto Berti Date: Mon, 9 Nov 2015 17:41:39 +0100 Subject: [PATCH] Treat async/await as anonymous token and deal with them in tokenize.py .. as cpython's parser does --- sphinx/pycode/pgen2/grammar.py | 2 -- sphinx/pycode/pgen2/tokenize.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/sphinx/pycode/pgen2/grammar.py b/sphinx/pycode/pgen2/grammar.py index 33643622bd..42e6d72ee1 100644 --- a/sphinx/pycode/pgen2/grammar.py +++ b/sphinx/pycode/pgen2/grammar.py @@ -165,8 +165,6 @@ opmap_raw = """ //= DOUBLESLASHEQUAL -> RARROW ... ELLIPSIS -async ASYNC -await AWAIT """ opmap = {} diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py index d625350500..c7013bf918 100644 --- a/sphinx/pycode/pgen2/tokenize.py +++ b/sphinx/pycode/pgen2/tokenize.py @@ -360,6 +360,16 @@ def generate_tokens(readline): spos, epos, pos = (lnum, start), (lnum, end), end token, initial = line[start:end], line[start] + if end < max: + next_pseudomatch = pseudoprog.match(line, end) + if next_pseudomatch: + n_start, n_end = next_pseudomatch.span(1) + n_token = line[n_start:n_end] + else: + n_token = None + else: + n_token = None + if initial in numchars or ( initial == '.' and token not in ('.', '...') ): # ordinary number @@ -396,6 +406,10 @@ def generate_tokens(readline): break else: # ordinary string yield (STRING, token, spos, epos, line) + elif token == 'await' and n_token: + yield (AWAIT, token, spos, epos, line) + elif token == 'async' and n_token in ('def', 'for', 'with'): + yield (ASYNC, token, spos, epos, line) elif initial in namechars: # ordinary name yield (NAME, token, spos, epos, line) elif token in ('...',): # ordinary name