mirror of
https://github.com/sphinx-doc/sphinx.git
synced 2026-09-03 20:52:55 -05:00
Treat async/await as anonymous token and deal with them in tokenize.py
.. as cpython's parser does
This commit is contained in:
@@ -165,8 +165,6 @@ opmap_raw = """
|
||||
//= DOUBLESLASHEQUAL
|
||||
-> RARROW
|
||||
... ELLIPSIS
|
||||
async ASYNC
|
||||
await AWAIT
|
||||
"""
|
||||
|
||||
opmap = {}
|
||||
|
||||
@@ -360,6 +360,16 @@ def generate_tokens(readline):
|
||||
spos, epos, pos = (lnum, start), (lnum, end), end
|
||||
token, initial = line[start:end], line[start]
|
||||
|
||||
if end < max:
|
||||
next_pseudomatch = pseudoprog.match(line, end)
|
||||
if next_pseudomatch:
|
||||
n_start, n_end = next_pseudomatch.span(1)
|
||||
n_token = line[n_start:n_end]
|
||||
else:
|
||||
n_token = None
|
||||
else:
|
||||
n_token = None
|
||||
|
||||
if initial in numchars or (
|
||||
initial == '.' and token not in ('.', '...')
|
||||
): # ordinary number
|
||||
@@ -396,6 +406,10 @@ def generate_tokens(readline):
|
||||
break
|
||||
else: # ordinary string
|
||||
yield (STRING, token, spos, epos, line)
|
||||
elif token == 'await' and n_token:
|
||||
yield (AWAIT, token, spos, epos, line)
|
||||
elif token == 'async' and n_token in ('def', 'for', 'with'):
|
||||
yield (ASYNC, token, spos, epos, line)
|
||||
elif initial in namechars: # ordinary name
|
||||
yield (NAME, token, spos, epos, line)
|
||||
elif token in ('...',): # ordinary name
|
||||
|
||||
Reference in New Issue
Block a user