Fixes from review

doc/usage/restructuredtext/directives.rst

@@ -1139,7 +1139,7 @@ derived forms), but provides enough to allow context-free grammars to be
 displayed in a way that causes uses of a symbol to be rendered as hyperlinks to
 the definition of the symbol.  There is this directive:
 
-.. rst:directive:: .. productionlist:: [tokenGroup]
+.. rst:directive:: .. productionlist:: [productionGroup]
 
    This directive is used to enclose a group of productions.  Each production
    is given on a single line and consists of a name, separated by a colon from
@@ -1147,9 +1147,9 @@ the definition of the symbol.  There is this directive:
    continuation line must begin with a colon placed at the same column as in
    the first line.
 
-   The ``tokenGroup`` argument to :rst:dir:`productionlist` serves to
+   The *productionGroup* argument to :rst:dir:`productionlist` serves to
    distinguish different sets of production lists that belong to different
-   grammars.  Multiple production lists with the same ``tokenGroup`` thus
+   grammars.  Multiple production lists with the same *productionGroup* thus
    define rules in the same scope.
 
    Blank lines are not allowed within ``productionlist`` directive arguments.
@@ -1157,14 +1157,14 @@ the definition of the symbol.  There is this directive:
    The definition can contain token names which are marked as interpreted text
    (e.g. "``sum ::= `integer` "+" `integer```") -- this generates
    cross-references to the productions of these tokens.  Outside of the
-   production list, you can reference token productions using
+   production list, you can reference to token productions using
    :rst:role:`token`.
-   However, if you have given a ``tokenGroup`` argument you must prefix the
+   However, if you have given a *productionGroup* argument you must prefix the
    token name in the cross-reference with the group name and a colon,
-   e.g., "``myTokenGroup:sum``" instead of just "``sum``".
-   If the token group should not be shown in the title of the link either an
-   explicit title can be given (e.g., "``myTitle <myTokenGroup:sum>``"),
-   or the target can be prefixed with a tilde (e.g., "``~myTokenGroup:sum``").
+   e.g., "``myGroup:sum``" instead of just "``sum``".
+   If the group should not be shown in the title of the link either
+   an explicit title can be given (e.g., "``myTitle <myGroup:sum>``"),
+   or the target can be prefixed with a tilde (e.g., "``~myGroup:sum``").
 
    Note that no further reST parsing is done in the production, so that you
    don't have to escape ``*`` or ``|`` characters.

sphinx/domains/std.py

@@ -406,7 +406,7 @@ class Glossary(SphinxDirective):
         return messages + [node]
 
 
-def token_xrefs(text: str, productionGroup: str) -> List[Node]:
+def token_xrefs(text: str, productionGroup: str = '') -> List[Node]:
     if len(productionGroup) != 0:
         productionGroup += ':'
     retnodes = []  # type: List[Node]
@@ -441,10 +441,7 @@ class ProductionList(SphinxDirective):
         node = addnodes.productionlist()  # type: Element
         # The backslash handling is from ObjectDescription.get_signatures
         nl_escape_re = re.compile(r'\\\n')
-        strip_backslash_re = re.compile(r'\\(.)')
         lines = nl_escape_re.sub('', self.arguments[0]).split('\n')
-        # remove backslashes to support (dummy) escapes; helps Vim highlighting
-        lines = [strip_backslash_re.sub(r'\1', line.strip()) for line in lines]
 
         productionGroup = ""
         i = 0
@@ -460,7 +457,10 @@ class ProductionList(SphinxDirective):
             subnode = addnodes.production(rule)
             subnode['tokenname'] = name.strip()
             if subnode['tokenname']:
-                idname = 'grammar-token2-%s_%s' \
+                # nodes.make_id converts '_' to '-',
+                # so we can use '_' to delimit group from name,
+                # and make sure we don't clash with other IDs.
+                idname = 'grammar-token-%s_%s' \
                          % (nodes.make_id(productionGroup), nodes.make_id(name))
                 if idname not in self.state.document.ids:
                     subnode['ids'].append(idname)

tests/roots/test-productionlist/index.rst

@@ -1,12 +1,12 @@
 .. toctree::
 
-	P1
-	P2
-	Bare
-	Dup1
-	Dup2
-	firstLineRule
-	LineContinuation
+   P1
+   P2
+   Bare
+   Dup1
+   Dup2
+   firstLineRule
+   LineContinuation
 
 - A: :token:`A`
 - B: :token:`B`

tests/test_build_html.py

@@ -222,7 +222,7 @@ def test_html4_output(app, status, warning):
          "[@class='reference internal']/code/span[@class='pre']", 'HOME'),
         (".//a[@href='#with']"
          "[@class='reference internal']/code/span[@class='pre']", '^with$'),
-        (".//a[@href='#grammar-token2-_try-stmt']"
+        (".//a[@href='#grammar-token-_try-stmt']"
          "[@class='reference internal']/code/span", '^statement$'),
         (".//a[@href='#some-label'][@class='reference internal']/span", '^here$'),
         (".//a[@href='#some-label'][@class='reference internal']/span", '^there$'),
@@ -254,7 +254,7 @@ def test_html4_output(app, status, warning):
         (".//dl/dt[@id='term-boson']", 'boson'),
         # a production list
         (".//pre/strong", 'try_stmt'),
-        (".//pre/a[@href='#grammar-token2-_try1-stmt']/code/span", 'try1_stmt'),
+        (".//pre/a[@href='#grammar-token-_try1-stmt']/code/span", 'try1_stmt'),
         # tests for ``only`` directive
         (".//p", 'A global substitution.'),
         (".//p", 'In HTML.'),

tests/test_domain_std.py

@@ -352,23 +352,23 @@ def test_productionlist(app, status, warning):
         linkText = span.text.strip()
         cases.append((text, link, linkText))
     assert cases == [
-        ('A', 'Bare.html#grammar-token2-_a', 'A'),
-        ('B', 'Bare.html#grammar-token2-_b', 'B'),
-        ('P1:A', 'P1.html#grammar-token2-p1_a', 'P1:A'),
-        ('P1:B', 'P1.html#grammar-token2-p1_b', 'P1:B'),
-        ('P2:A', 'P1.html#grammar-token2-p1_a', 'P1:A'),
-        ('P2:B', 'P2.html#grammar-token2-p2_b', 'P2:B'),
-        ('Explicit title A, plain', 'Bare.html#grammar-token2-_a', 'MyTitle'),
-        ('Explicit title A, colon', 'Bare.html#grammar-token2-_a', 'My:Title'),
-        ('Explicit title P1:A, plain', 'P1.html#grammar-token2-p1_a', 'MyTitle'),
-        ('Explicit title P1:A, colon', 'P1.html#grammar-token2-p1_a', 'My:Title'),
-        ('Tilde A', 'Bare.html#grammar-token2-_a', 'A'),
-        ('Tilde P1:A', 'P1.html#grammar-token2-p1_a', 'A'),
-        ('Tilde explicit title P1:A', 'P1.html#grammar-token2-p1_a', '~MyTitle'),
-        ('Tilde, explicit title P1:A', 'P1.html#grammar-token2-p1_a', 'MyTitle'),
-        ('Dup', 'Dup2.html#grammar-token2-_dup', 'Dup'),
-        ('FirstLine', 'firstLineRule.html#grammar-token2-_firstline', 'FirstLine'),
-        ('SecondLine', 'firstLineRule.html#grammar-token2-_secondline', 'SecondLine'),
+        ('A', 'Bare.html#grammar-token-_a', 'A'),
+        ('B', 'Bare.html#grammar-token-_b', 'B'),
+        ('P1:A', 'P1.html#grammar-token-p1_a', 'P1:A'),
+        ('P1:B', 'P1.html#grammar-token-p1_b', 'P1:B'),
+        ('P2:A', 'P1.html#grammar-token-p1_a', 'P1:A'),
+        ('P2:B', 'P2.html#grammar-token-p2_b', 'P2:B'),
+        ('Explicit title A, plain', 'Bare.html#grammar-token-_a', 'MyTitle'),
+        ('Explicit title A, colon', 'Bare.html#grammar-token-_a', 'My:Title'),
+        ('Explicit title P1:A, plain', 'P1.html#grammar-token-p1_a', 'MyTitle'),
+        ('Explicit title P1:A, colon', 'P1.html#grammar-token-p1_a', 'My:Title'),
+        ('Tilde A', 'Bare.html#grammar-token-_a', 'A'),
+        ('Tilde P1:A', 'P1.html#grammar-token-p1_a', 'A'),
+        ('Tilde explicit title P1:A', 'P1.html#grammar-token-p1_a', '~MyTitle'),
+        ('Tilde, explicit title P1:A', 'P1.html#grammar-token-p1_a', 'MyTitle'),
+        ('Dup', 'Dup2.html#grammar-token-_dup', 'Dup'),
+        ('FirstLine', 'firstLineRule.html#grammar-token-_firstline', 'FirstLine'),
+        ('SecondLine', 'firstLineRule.html#grammar-token-_secondline', 'SecondLine'),
     ]
 
     text = (app.outdir / 'LineContinuation.html').text()