Add Python license info, add parse.c source generated by Cython.

LICENSE

@@ -1,17 +1,19 @@
-Copyright (c) 2007-2008 by the respective authors (see AUTHORS file).
+Copyright (c) 2007-2008 by the Sphinx team (see AUTHORS file).  All
-All rights reserved.
+rights reserved.
+
+License for Sphinx
+==================
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
 met:
 
-    * Redistributions of source code must retain the above copyright
+* Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
+  notice, this list of conditions and the following disclaimer.
 
-    * Redistributions in binary form must reproduce the above
+* Redistributions in binary form must reproduce the above copyright
-      copyright notice, this list of conditions and the following
+  notice, this list of conditions and the following disclaimer in the
-      disclaimer in the documentation and/or other materials provided
+  documentation and/or other materials provided with the distribution.
-      with the distribution.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
@@ -24,3 +26,58 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Licenses for incorporated software
+==================================
+
+The pgen2 package, included in this distribution under the name
+sphinx.pycode.pgen2, is available in the Python 2.6 distribution under
+the PSF license agreement for Python:
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+   ("PSF"), and the Individual or Organization ("Licensee") accessing
+   and otherwise using Python 2.6 software in source or binary form
+   and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF
+   hereby grants Licensee a nonexclusive, royalty-free, world-wide
+   license to reproduce, analyze, test, perform and/or display
+   publicly, prepare derivative works, distribute, and otherwise use
+   Python 2.6 alone or in any derivative version, provided, however,
+   that PSF's License Agreement and PSF's notice of copyright, i.e.,
+   "Copyright © 2001-2008 Python Software Foundation; All Rights
+   Reserved" are retained in Python 2.6 alone or in any derivative
+   version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+   or incorporates Python 2.6 or any part thereof, and wants to make
+   the derivative work available to others as provided herein, then
+   Licensee hereby agrees to include in any such work a brief summary
+   of the changes made to Python 2.6.
+
+4. PSF is making Python 2.6 available to Licensee on an "AS IS" basis.
+   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY
+   WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY
+   REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY
+   PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 2.6 WILL NOT INFRINGE
+   ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+   2.6 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+   AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON
+   2.6, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY
+   THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+   breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+   relationship of agency, partnership, or joint venture between PSF
+   and Licensee.  This License Agreement does not grant permission to
+   use PSF trademarks or trade name in a trademark sense to endorse or
+   promote products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python 2.6, Licensee
+   agrees to be bound by the terms and conditions of this License
+   Agreement.

sphinx/pycode/__init__.py

@@ -5,7 +5,7 @@
 
     Utilities parsing and analyzing Python code.
 
-    :copyright: 2008 by Georg Brandl.
+    :copyright: 2008-2009 by Georg Brandl.
     :license: BSD, see LICENSE for details.
 """
 
@@ -13,7 +13,7 @@ import sys
 from os import path
 from cStringIO import StringIO
 
-from sphinx.pycode import pytree
+from sphinx.pycode import nodes
 from sphinx.pycode.pgen2 import driver, token, tokenize, parse, literals
 from sphinx.util.docstrings import prepare_docstring, prepare_commentdoc
 
@@ -21,7 +21,7 @@ from sphinx.util.docstrings import prepare_docstring, prepare_commentdoc
 # load the Python grammar
 _grammarfile = path.join(path.dirname(__file__), 'Grammar.txt')
 pygrammar = driver.load_grammar(_grammarfile)
-pydriver = driver.Driver(pygrammar, convert=pytree.convert)
+pydriver = driver.Driver(pygrammar, convert=nodes.convert)
 
 # an object with attributes corresponding to token and symbol names
 class sym: pass
@@ -35,10 +35,10 @@ number2name = pygrammar.number2symbol.copy()
 number2name.update(token.tok_name)
 
 
-_eq = pytree.Leaf(token.EQUAL, '=')
+_eq = nodes.Leaf(token.EQUAL, '=')
 
 
-class AttrDocVisitor(pytree.NodeVisitor):
+class AttrDocVisitor(nodes.NodeVisitor):
     """
     Visitor that collects docstrings for attribute assignments on toplevel and
     in classes.
@@ -263,5 +263,5 @@ if __name__ == '__main__':
     #    print '\n'.join(doc)
     pprint.pprint(ma.find_tags())
     x3 = time.time()
-    #print pytree.nice_repr(ma.parsetree, number2name)
+    #print nodes.nice_repr(ma.parsetree, number2name)
     print "tokenizing %.4f, parsing %.4f, finding %.4f" % (x1-x0, x2-x1, x3-x2)

sphinx/pycode/nodes.py

@@ -1,80 +1,47 @@
-# Copyright 2006 Google, Inc. All Rights Reserved.
+# -*- coding: utf-8 -*-
-# Licensed to PSF under a Contributor Agreement.
+"""
+    sphinx.pycode.nodes
+    ~~~~~~~~~~~~~~~~~~~
 
-"""Python parse tree definitions.
+    Parse tree node implementations.
 
-This is a very concrete parse tree; we need to keep every token and
+    :copyright: 2009 by Georg Brandl.
-even the comments and whitespace between tokens.
+    :license: BSD, see LICENSE for details.
-
-Adapted for read-only nodes from pytree.py in Python's 2to3 tool, and
-added a few bits.
 """
 
-__author__ = "Guido van Rossum <guido@python.org>"
 
-
+class BaseNode(object):
-class Base(object):
+    """
-
+    Node superclass for both terminal and nonterminal nodes.
-    """Abstract base class for Node and Leaf.
-
-    This provides some default functionality and boilerplate using the
-    template pattern.
-
-    A node may be a subnode of at most one parent.
     """
 
-    # Default values for instance variables
+    def _eq(self, other):
-    type = None    # int: token number (< 256) or symbol number (>= 256)
+        raise NotImplementedError
-    parent = None  # Parent node pointer, or None
-    children = ()  # Tuple of subnodes
-    was_changed = False
 
     def __eq__(self, other):
-        """Compares two nodes for equality.
-
-        This calls the method _eq().
-        """
         if self.__class__ is not other.__class__:
             return NotImplemented
         return self._eq(other)
 
     def __ne__(self, other):
-        """Compares two nodes for inequality.
-
-        This calls the method _eq().
-        """
         if self.__class__ is not other.__class__:
             return NotImplemented
         return not self._eq(other)
 
-    def _eq(self, other):
+    def get_prev_sibling(self):
-        """Compares two nodes for equality.
+        """Return previous child in parent's children, or None."""
-
-        This is called by __eq__ and __ne__.  It is only called if the
-        two nodes have the same type.  This must be implemented by the
-        concrete subclass.  Nodes should be considered equal if they
-        have the same structure, ignoring the prefix string and other
-        context information.
-        """
-        raise NotImplementedError
-
-    def get_lineno(self):
-        """Returns the line number which generated the invocant node."""
-        node = self
-        while not isinstance(node, Leaf):
-            if not node.children:
-                return
-            node = node.children[0]
-        return node.lineno
-
-    def get_next_sibling(self):
-        """Return the node immediately following the invocant in their
-        parent's children list. If the invocant does not have a next
-        sibling, return None."""
         if self.parent is None:
             return None
+        for i, child in enumerate(self.parent.children):
+            if child is self:
+                if i == 0:
+                    return None
+                return self.parent.children[i-1]
 
-        # Can't use index(); we need to test by identity
+    def get_next_sibling(self):
+        """Return next child in parent's children, or None."""
+        if self.parent is None:
+            return None
         for i, child in enumerate(self.parent.children):
             if child is self:
                 try:
@@ -82,20 +49,6 @@ class Base(object):
                 except IndexError:
                     return None
 
-    def get_prev_sibling(self):
-        """Return the node immediately preceding the invocant in their
-        parent's children list. If the invocant does not have a previous
-        sibling, return None."""
-        if self.parent is None:
-            return None
-
-        # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
-            if child is self:
-                if i == 0:
-                    return None
-                return self.parent.children[i-1]
-
     def get_prev_leaf(self):
         """Return the leaf node that precedes this node in the parse tree."""
         def last_child(node):
@@ -114,46 +67,51 @@ class Base(object):
             return last_child(prev)
         return self.parent.get_prev_leaf()
 
-    def get_suffix(self):
+    def get_next_leaf(self):
-        """Return the string immediately following the invocant node. This
+        """Return self if leaf, otherwise the leaf node that succeeds this
-        is effectively equivalent to node.get_next_sibling().get_prefix()"""
+        node in the parse tree.
-        next_sib = self.get_next_sibling()
+        """
-        if next_sib is None:
+        node = self
-            return ""
+        while not isinstance(node, Leaf):
-        return next_sib.get_prefix()
+            assert node.children
+            node = node.children[0]
+        return node
+
+    def get_lineno(self):
+        """Return the line number which generated the invocant node."""
+        return self.get_next_leaf().lineno
+
+    def get_prefix(self):
+        """Return the prefix of the next leaf node."""
+        # only leaves carry a prefix
+        return self.get_next_leaf().prefix
 
 
-class Node(Base):
+class Node(BaseNode):
-
+    """
-    """Concrete implementation for interior nodes."""
+    Node implementation for nonterminals.
+    """
 
     def __init__(self, type, children, context=None):
-        """Initializer.
+        # type of nonterminals is >= 256
-
-        Takes a type constant (a symbol number >= 256), a sequence of
-        child nodes, and an optional context keyword argument.
-
-        As a side effect, the parent pointers of the children are updated.
-        """
         # assert type >= 256, type
         self.type = type
         self.children = list(children)
         for ch in self.children:
             # assert ch.parent is None, repr(ch)
             ch.parent = self
-        # if prefix is not None:
-        #     self.set_prefix(prefix)
 
     def __repr__(self):
-        return "%s(%s, %r)" % (self.__class__.__name__,
+        return '%s(%s, %r)' % (self.__class__.__name__, self.type, self.children)
-                               self.type, self.children)
 
     def __str__(self):
         """This reproduces the input source exactly."""
-        return "".join(map(str, self.children))
+        return ''.join(map(str, self.children))
 
-    def compact(self):
+    def _eq(self, other):
-        return ''.join(child.compact() for child in self.children)
+        return (self.type, self.children) == (other.type, other.children)
+
+    # support indexing the node directly instead of .children
 
     def __getitem__(self, index):
         return self.children[index]
@@ -164,84 +122,35 @@ class Node(Base):
     def __len__(self):
         return len(self.children)
 
-    def _eq(self, other):
-        """Compares two nodes for equality."""
-        return (self.type, self.children) == (other.type, other.children)
 
-    def post_order(self):
+class Leaf(BaseNode):
-        """Returns a post-order iterator for the tree."""
+    """
-        for child in self.children:
+    Node implementation for leaf nodes (terminals).
-            for node in child.post_order():
+    """
-                yield node
+    prefix = ''  # Whitespace and comments preceding this token in the input
-        yield self
-
-    def pre_order(self):
-        """Returns a pre-order iterator for the tree."""
-        yield self
-        for child in self.children:
-            for node in child.post_order():
-                yield node
-
-    def get_prefix(self):
-        """Returns the prefix for the node.
-
-        This passes the call on to the first child.
-        """
-        if not self.children:
-            return ""
-        return self.children[0].get_prefix()
-
-
-class Leaf(Base):
-
-    """Concrete implementation for leaf nodes."""
-
-    # Default values for instance variables
-    prefix = ""  # Whitespace and comments preceding this token in the input
     lineno = 0   # Line where this token starts in the input
     column = 0   # Column where this token tarts in the input
 
     def __init__(self, type, value, context=None):
-        """Initializer.
+        # type of terminals is below 256
-
-        Takes a type constant (a token number < 256), a string value,
-        and an optional context keyword argument.
-        """
         # assert 0 <= type < 256, type
-        if context is not None:
-            self.prefix, (self.lineno, self.column) = context
         self.type = type
         self.value = value
-        # if prefix is not None:
+        if context is not None:
-        #     self.prefix = prefix
+            self.prefix, (self.lineno, self.column) = context
 
     def __repr__(self):
-        return "%s(%r, %r, %r)" % (self.__class__.__name__,
+        return '%s(%r, %r, %r)' % (self.__class__.__name__,
                                    self.type, self.value, self.prefix)
 
     def __str__(self):
         """This reproduces the input source exactly."""
         return self.prefix + str(self.value)
 
-    def compact(self):
-        return str(self.value)
-
     def _eq(self, other):
         """Compares two nodes for equality."""
         return (self.type, self.value) == (other.type, other.value)
 
-    def post_order(self):
-        """Returns a post-order iterator for the tree."""
-        yield self
-
-    def pre_order(self):
-        """Returns a pre-order iterator for the tree."""
-        yield self
-
-    def get_prefix(self):
-        """Returns the prefix for the node."""
-        return self.prefix
-
 
 def convert(grammar, raw_node):
     """Convert raw node to a Node or Leaf instance."""

sphinx/pycode/pgen2/parse.pyx

@@ -1,6 +1,8 @@
 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
 # Licensed to PSF under a Contributor Agreement.
 
+# Adapted from parse.py to be compiled with Cython by Georg Brandl.
+
 """Parser engine for the grammar tables generated by pgen.
 
 The grammar table must be loaded first.
@@ -10,7 +12,7 @@ how this parsing engine works.
 
 """
 
-from sphinx.pycode.pytree import Node, Leaf
+from sphinx.pycode.nodes import Node, Leaf
 
 DEF NAME = 1