Tools/compiler/astgen.py

   1 """Generate ast module from specification
   2
   3 This script generates the ast module from a simple specification,
   4 which makes it easy to accomodate changes in the grammar.  This
   5 approach would be quite reasonable if the grammar changed often.
   6 Instead, it is rather complex to generate the appropriate code.  And
   7 the Node interface has changed more often than the grammar.
   8 """
   9
  10 import fileinput
  11 import re
  12 import sys
  13 from StringIO import StringIO
  14
  15 SPEC = "ast.txt"
  16 COMMA = ", "
  17
  18 def load_boilerplate(file):
  19     f = open(file)
  20     buf = f.read()
  21     f.close()
  22     i = buf.find('### ''PROLOGUE')
  23     j = buf.find('### ''EPILOGUE')
  24     pro = buf[i+12:j].strip()
  25     epi = buf[j+12:].strip()
  26     return pro, epi
  27
  28 def strip_default(arg):
  29     """Return the argname from an 'arg = default' string"""
  30     i = arg.find('=')
  31     if i == -1:
  32         return arg
  33     t = arg[:i].strip()
  34     return t
  35
  36 P_NODE = 1
  37 P_OTHER = 2
  38 P_NESTED = 3
  39 P_NONE = 4
  40
  41 class NodeInfo:
  42     """Each instance describes a specific AST node"""
  43     def __init__(self, name, args):
  44         self.name = name
  45         self.args = args.strip()
  46         self.argnames = self.get_argnames()
  47         self.argprops = self.get_argprops()
  48         self.nargs = len(self.argnames)
  49         self.init = []
  50
  51     def get_argnames(self):
  52         if '(' in self.args:
  53             i = self.args.find('(')
  54             j = self.args.rfind(')')
  55             args = self.args[i+1:j]
  56         else:
  57             args = self.args
  58         return [strip_default(arg.strip())
  59                 for arg in args.split(',') if arg]
  60
  61     def get_argprops(self):
  62         """Each argument can have a property like '*' or '!'
  63
  64         XXX This method modifies the argnames in place!
  65         """
  66         d = {}
  67         hardest_arg = P_NODE
  68         for i in range(len(self.argnames)):
  69             arg = self.argnames[i]
  70             if arg.endswith('*'):
  71                 arg = self.argnames[i] = arg[:-1]
  72                 d[arg] = P_OTHER
  73                 hardest_arg = max(hardest_arg, P_OTHER)
  74             elif arg.endswith('!'):
  75                 arg = self.argnames[i] = arg[:-1]
  76                 d[arg] = P_NESTED
  77                 hardest_arg = max(hardest_arg, P_NESTED)
  78             elif arg.endswith('&'):
  79                 arg = self.argnames[i] = arg[:-1]
  80                 d[arg] = P_NONE
  81                 hardest_arg = max(hardest_arg, P_NONE)
  82             else:
  83                 d[arg] = P_NODE
  84         self.hardest_arg = hardest_arg
  85
  86         if hardest_arg > P_NODE:
  87             self.args = self.args.replace('*', '')
  88             self.args = self.args.replace('!', '')
  89             self.args = self.args.replace('&', '')
  90
  91         return d
  92
  93     def gen_source(self):
  94         buf = StringIO()
  95         print >> buf, "class %s(Node):" % self.name
  96         self._gen_init(buf)
  97         print >> buf
  98         self._gen_getChildren(buf)
  99         print >> buf
 100         self._gen_getChildNodes(buf)
 101         print >> buf
 102         self._gen_repr(buf)
 103         buf.seek(0, 0)
 104         return buf.read()
 105
 106     def _gen_init(self, buf):
 107         if self.args:
 108             print >> buf, "    def __init__(self, %s, lineno=None):" % self.args
 109         else:
 110             print >> buf, "    def __init__(self, lineno=None):"
 111         if self.argnames:
 112             for name in self.argnames:
 113                 print >> buf, "        self.%s = %s" % (name, name)
 114         print >> buf, "        self.lineno = lineno"
 115         # Copy the lines in self.init, indented four spaces.  The rstrip()
 116         # business is to get rid of the four spaces if line happens to be
 117         # empty, so that reindent.py is happy with the output.
 118         for line in self.init:
 119             print >> buf, ("    " + line).rstrip()
 120
 121     def _gen_getChildren(self, buf):
 122         print >> buf, "    def getChildren(self):"
 123         if len(self.argnames) == 0:
 124             print >> buf, "        return ()"
 125         else:
 126             if self.hardest_arg < P_NESTED:
 127                 clist = COMMA.join(["self.%s" % c
 128                                     for c in self.argnames])
 129                 if self.nargs == 1:
 130                     print >> buf, "        return %s," % clist
 131                 else:
 132                     print >> buf, "        return %s" % clist
 133             else:
 134                 if len(self.argnames) == 1:
 135                     print >> buf, "        return tuple(flatten(self.%s))" % self.argnames[0]
 136                 else:
 137                     print >> buf, "        children = []"
 138                     template = "        children.%s(%sself.%s%s)"
 139                     for name in self.argnames:
 140                         if self.argprops[name] == P_NESTED:
 141                             print >> buf, template % ("extend", "flatten(",
 142                                                       name, ")")
 143                         else:
 144                             print >> buf, template % ("append", "", name, "")
 145                     print >> buf, "        return tuple(children)"
 146
 147     def _gen_getChildNodes(self, buf):
 148         print >> buf, "    def getChildNodes(self):"
 149         if len(self.argnames) == 0:
 150             print >> buf, "        return ()"
 151         else:
 152             if self.hardest_arg < P_NESTED:
 153                 clist = ["self.%s" % c
 154                          for c in self.argnames
 155                          if self.argprops[c] == P_NODE]
 156                 if len(clist) == 0:
 157                     print >> buf, "        return ()"
 158                 elif len(clist) == 1:
 159                     print >> buf, "        return %s," % clist[0]
 160                 else:
 161                     print >> buf, "        return %s" % COMMA.join(clist)
 162             else:
 163                 print >> buf, "        nodelist = []"
 164                 template = "        nodelist.%s(%sself.%s%s)"
 165                 for name in self.argnames:
 166                     if self.argprops[name] == P_NONE:
 167                         tmp = ("        if self.%s is not None:\n"
 168                                "            nodelist.append(self.%s)")
 169                         print >> buf, tmp % (name, name)
 170                     elif self.argprops[name] == P_NESTED:
 171                         print >> buf, template % ("extend", "flatten_nodes(",
 172                                                   name, ")")
 173                     elif self.argprops[name] == P_NODE:
 174                         print >> buf, template % ("append", "", name, "")
 175                 print >> buf, "        return tuple(nodelist)"
 176
 177     def _gen_repr(self, buf):
 178         print >> buf, "    def __repr__(self):"
 179         if self.argnames:
 180             fmt = COMMA.join(["%s"] * self.nargs)
 181             if '(' in self.args:
 182                 fmt = '(%s)' % fmt
 183             vals = ["repr(self.%s)" % name for name in self.argnames]
 184             vals = COMMA.join(vals)
 185             if self.nargs == 1:
 186                 vals = vals + ","
 187             print >> buf, '        return "%s(%s)" %% (%s)' % \
 188                   (self.name, fmt, vals)
 189         else:
 190             print >> buf, '        return "%s()"' % self.name
 191
 192 rx_init = re.compile('init\((.*)\):')
 193
 194 def parse_spec(file):
 195     classes = {}
 196     cur = None
 197     for line in fileinput.input(file):
 198         if line.strip().startswith('#'):
 199             continue
 200         mo = rx_init.search(line)
 201         if mo is None:
 202             if cur is None:
 203                 # a normal entry
 204                 try:
 205                     name, args = line.split(':')
 206                 except ValueError:
 207                     continue
 208                 classes[name] = NodeInfo(name, args)
 209                 cur = None
 210             else:
 211                 # some code for the __init__ method
 212                 cur.init.append(line)
 213         else:
 214             # some extra code for a Node's __init__ method
 215             name = mo.group(1)
 216             cur = classes[name]
 217     return sorted(classes.values(), key=lambda n: n.name)
 218
 219 def main():
 220     prologue, epilogue = load_boilerplate(sys.argv[-1])
 221     print prologue
 222     print
 223     classes = parse_spec(SPEC)
 224     for info in classes:
 225         print info.gen_source()
 226     print epilogue
 227
 228 if __name__ == "__main__":
 229     main()
 230     sys.exit(0)
 231
 232 ### PROLOGUE
 233 """Python abstract syntax node definitions
 234
 235 This file is automatically generated by Tools/compiler/astgen.py
 236 """
 237 from consts import CO_VARARGS, CO_VARKEYWORDS
 238
 239 def flatten(seq):
 240     l = []
 241     for elt in seq:
 242         t = type(elt)
 243         if t is tuple or t is list:
 244             for elt2 in flatten(elt):
 245                 l.append(elt2)
 246         else:
 247             l.append(elt)
 248     return l
 249
 250 def flatten_nodes(seq):
 251     return [n for n in flatten(seq) if isinstance(n, Node)]
 252
 253 nodes = {}
 254
 255 class Node:
 256     """Abstract base class for ast nodes."""
 257     def getChildren(self):
 258         pass # implemented by subclasses
 259     def __iter__(self):
 260         for n in self.getChildren():
 261             yield n
 262     def asList(self): # for backwards compatibility
 263         return self.getChildren()
 264     def getChildNodes(self):
 265         pass # implemented by subclasses
 266
 267 class EmptyNode(Node):
 268     pass
 269
 270 class Expression(Node):
 271     # Expression is an artificial node class to support "eval"
 272     nodes["expression"] = "Expression"
 273     def __init__(self, node):
 274         self.node = node
 275
 276     def getChildren(self):
 277         return self.node,
 278
 279     def getChildNodes(self):
 280         return self.node,
 281
 282     def __repr__(self):
 283         return "Expression(%s)" % (repr(self.node))
 284
 285 ### EPILOGUE
 286 for name, obj in globals().items():
 287     if isinstance(obj, type) and issubclass(obj, Node):
 288         nodes[name.lower()] = obj