2 """Generate C code from an ASDL description."""
5 # handle fields that have a type but no name
7 import os
, sys
, traceback
15 """Return a string for the C name of the type.
17 This function special cases the default types provided by asdl:
18 identifier, string, int, bool.
20 # XXX ack! need to figure out where Id is useful and where string
21 if isinstance(name
, asdl
.Id
):
23 if name
in asdl
.builtin_types
:
28 def reflow_lines(s
, depth
):
29 """Reflow the line s indented depth tabs.
31 Return a sequence of lines where no line extends beyond MAX_COL
32 when properly indented. The first line is properly indented based
33 exclusively on depth * TABSIZE. All following lines -- these are
34 the reflowed lines generated by this function -- start at the same
35 column as the first character beyond the opening { in the first
38 size
= MAX_COL
- depth
* TABSIZE
45 while len(cur
) > size
:
46 i
= cur
.rfind(' ', 0, size
)
47 # XXX this should be fixed for real
48 if i
== -1 and 'GeneratorExp' in cur
:
50 assert i
!= -1, "Impossible line %d to reflow: %r" % (size
, s
)
51 lines
.append(padding
+ cur
[:i
])
53 # find new size based on brace
54 j
= cur
.find('{', 0, i
)
56 j
+= 2 # account for the brace and the space after it
60 j
= cur
.find('(', 0, i
)
62 j
+= 1 # account for the paren (no space after it)
67 lines
.append(padding
+ cur
)
71 """Return True if a sum is a simple.
73 A sum is simple if its types have no fields, e.g.
74 unaryop = Invert | Not | UAdd | USub
82 class EmitVisitor(asdl
.VisitorBase
):
83 """Visit that emits lines"""
85 def __init__(self
, file):
87 super(EmitVisitor
, self
).__init
__()
89 def emit(self
, s
, depth
, reflow
=1):
90 # XXX reflow long lines?
92 lines
= reflow_lines(s
, depth
)
96 line
= (" " * TABSIZE
* depth
) + line
+ "\n"
99 class TypeDefVisitor(EmitVisitor
):
100 def visitModule(self
, mod
):
104 def visitType(self
, type, depth
=0):
105 self
.visit(type.value
, type.name
, depth
)
107 def visitSum(self
, sum, name
, depth
):
109 self
.simple_sum(sum, name
, depth
)
111 self
.sum_with_constructors(sum, name
, depth
)
113 def simple_sum(self
, sum, name
, depth
):
115 for i
in range(len(sum.types
)):
117 enum
.append("%s=%d" % (type.name
, i
+ 1))
118 enums
= ", ".join(enum
)
119 ctype
= get_c_type(name
)
120 s
= "typedef enum _%s { %s } %s;" % (name
, enums
, ctype
)
124 def sum_with_constructors(self
, sum, name
, depth
):
125 ctype
= get_c_type(name
)
126 s
= "typedef struct _%(name)s *%(ctype)s;" % locals()
130 def visitProduct(self
, product
, name
, depth
):
131 ctype
= get_c_type(name
)
132 s
= "typedef struct _%(name)s *%(ctype)s;" % locals()
136 class StructVisitor(EmitVisitor
):
137 """Visitor to generate typdefs for AST."""
139 def visitModule(self
, mod
):
143 def visitType(self
, type, depth
=0):
144 self
.visit(type.value
, type.name
, depth
)
146 def visitSum(self
, sum, name
, depth
):
147 if not is_simple(sum):
148 self
.sum_with_constructors(sum, name
, depth
)
150 def sum_with_constructors(self
, sum, name
, depth
):
151 def emit(s
, depth
=depth
):
152 self
.emit(s
% sys
._getframe
(1).f_locals
, depth
)
154 for i
in range(len(sum.types
)):
156 enum
.append("%s_kind=%d" % (type.name
, i
+ 1))
158 emit("enum _%(name)s_kind {" + ", ".join(enum
) + "};")
160 emit("struct _%(name)s {")
161 emit("enum _%(name)s_kind kind;", depth
+ 1)
162 emit("union {", depth
+ 1)
164 self
.visit(t
, depth
+ 2)
165 emit("} v;", depth
+ 1)
166 for field
in sum.attributes
:
167 # rudimentary attribute handling
168 type = str(field
.type)
169 assert type in asdl
.builtin_types
, type
170 emit("%s %s;" % (type, field
.name
), depth
+ 1);
174 def visitConstructor(self
, cons
, depth
):
176 self
.emit("struct {", depth
)
177 for f
in cons
.fields
:
178 self
.visit(f
, depth
+ 1)
179 self
.emit("} %s;" % cons
.name
, depth
)
182 # XXX not sure what I want here, nothing is probably fine
185 def visitField(self
, field
, depth
):
186 # XXX need to lookup field.type, because it might be something
188 ctype
= get_c_type(field
.type)
191 if field
.type.value
in ('cmpop',):
192 self
.emit("asdl_int_seq *%(name)s;" % locals(), depth
)
194 self
.emit("asdl_seq *%(name)s;" % locals(), depth
)
196 self
.emit("%(ctype)s %(name)s;" % locals(), depth
)
198 def visitProduct(self
, product
, name
, depth
):
199 self
.emit("struct _%(name)s {" % locals(), depth
)
200 for f
in product
.fields
:
201 self
.visit(f
, depth
+ 1)
202 self
.emit("};", depth
)
205 class PrototypeVisitor(EmitVisitor
):
206 """Generate function prototypes for the .h file"""
208 def visitModule(self
, mod
):
212 def visitType(self
, type):
213 self
.visit(type.value
, type.name
)
215 def visitSum(self
, sum, name
):
220 self
.visit(t
, name
, sum.attributes
)
222 def get_args(self
, fields
):
223 """Return list of C argument into, one for each field.
225 Argument info is 3-tuple of a C type, variable name, and flag
226 that is true if type can be NULL.
233 c
= unnamed
[name
] = unnamed
.get(name
, 0) + 1
235 name
= "name%d" % (c
- 1)
238 # XXX should extend get_c_type() to handle this
240 if f
.type.value
in ('cmpop',):
241 ctype
= "asdl_int_seq *"
245 ctype
= get_c_type(f
.type)
246 args
.append((ctype
, name
, f
.opt
or f
.seq
))
249 def visitConstructor(self
, cons
, type, attrs
):
250 args
= self
.get_args(cons
.fields
)
251 attrs
= self
.get_args(attrs
)
252 ctype
= get_c_type(type)
253 self
.emit_function(cons
.name
, ctype
, args
, attrs
)
255 def emit_function(self
, name
, ctype
, args
, attrs
, union
=1):
258 argstr
= ", ".join(["%s %s" % (atype
, aname
)
259 for atype
, aname
, opt
in args
])
260 argstr
+= ", PyArena *arena"
262 argstr
= "PyArena *arena"
264 for i
in range(1, len(args
)+1):
266 self
.emit("#define %s(%s) _Py_%s(%s)" % (name
, margs
, name
, margs
), 0,
268 self
.emit("%s _Py_%s(%s);" % (ctype
, name
, argstr
), 0)
270 def visitProduct(self
, prod
, name
):
271 self
.emit_function(name
, get_c_type(name
),
272 self
.get_args(prod
.fields
), [], union
=0)
274 class FunctionVisitor(PrototypeVisitor
):
275 """Visitor to generate constructor functions for AST."""
277 def emit_function(self
, name
, ctype
, args
, attrs
, union
=1):
278 def emit(s
, depth
=0, reflow
=1):
279 self
.emit(s
, depth
, reflow
)
280 argstr
= ", ".join(["%s %s" % (atype
, aname
)
281 for atype
, aname
, opt
in args
+ attrs
])
283 argstr
+= ", PyArena *arena"
285 argstr
= "PyArena *arena"
286 self
.emit("%s" % ctype
, 0)
287 emit("%s(%s)" % (name
, argstr
))
289 emit("%s p;" % ctype
, 1)
290 for argtype
, argname
, opt
in args
:
291 # XXX hack alert: false is allowed for a bool
292 if not opt
and not (argtype
== "bool" or argtype
== "int"):
293 emit("if (!%s) {" % argname
, 1)
294 emit("PyErr_SetString(PyExc_ValueError,", 2)
295 msg
= "field %s is required for %s" % (argname
, name
)
296 emit(' "%s");' % msg
,
298 emit('return NULL;', 2)
301 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype
, 1);
303 emit("return NULL;", 2)
305 self
.emit_body_union(name
, args
, attrs
)
307 self
.emit_body_struct(name
, args
, attrs
)
312 def emit_body_union(self
, name
, args
, attrs
):
313 def emit(s
, depth
=0, reflow
=1):
314 self
.emit(s
, depth
, reflow
)
315 emit("p->kind = %s_kind;" % name
, 1)
316 for argtype
, argname
, opt
in args
:
317 emit("p->v.%s.%s = %s;" % (name
, argname
, argname
), 1)
318 for argtype
, argname
, opt
in attrs
:
319 emit("p->%s = %s;" % (argname
, argname
), 1)
321 def emit_body_struct(self
, name
, args
, attrs
):
322 def emit(s
, depth
=0, reflow
=1):
323 self
.emit(s
, depth
, reflow
)
324 for argtype
, argname
, opt
in args
:
325 emit("p->%s = %s;" % (argname
, argname
), 1)
328 class PickleVisitor(EmitVisitor
):
330 def visitModule(self
, mod
):
334 def visitType(self
, type):
335 self
.visit(type.value
, type.name
)
337 def visitSum(self
, sum, name
):
340 def visitProduct(self
, sum, name
):
343 def visitConstructor(self
, cons
, name
):
346 def visitField(self
, sum):
349 class MarshalPrototypeVisitor(PickleVisitor
):
351 def prototype(self
, sum, name
):
352 ctype
= get_c_type(name
)
353 self
.emit("static int marshal_write_%s(PyObject **, int *, %s);"
356 visitProduct
= visitSum
= prototype
358 class PyTypesDeclareVisitor(PickleVisitor
):
360 def visitProduct(self
, prod
, name
):
361 self
.emit("static PyTypeObject *%s_type;" % name
, 0)
362 self
.emit("static PyObject* ast2obj_%s(void*);" % name
, 0)
364 self
.emit("static char *%s_fields[]={" % name
,0)
365 for f
in prod
.fields
:
366 self
.emit('"%s",' % f
.name
, 1)
369 def visitSum(self
, sum, name
):
370 self
.emit("static PyTypeObject *%s_type;" % name
, 0)
372 self
.emit("static char *%s_attributes[] = {" % name
, 0)
373 for a
in sum.attributes
:
374 self
.emit('"%s",' % a
.name
, 1)
378 ptype
= get_c_type(name
)
381 tnames
.append(str(t
.name
)+"_singleton")
382 tnames
= ", *".join(tnames
)
383 self
.emit("static PyObject *%s;" % tnames
, 0)
384 self
.emit("static PyObject* ast2obj_%s(%s);" % (name
, ptype
), 0)
386 self
.visitConstructor(t
, name
)
388 def visitConstructor(self
, cons
, name
):
389 self
.emit("static PyTypeObject *%s_type;" % cons
.name
, 0)
391 self
.emit("static char *%s_fields[]={" % cons
.name
, 0)
392 for t
in cons
.fields
:
393 self
.emit('"%s",' % t
.name
, 1)
396 class PyTypesVisitor(PickleVisitor
):
398 def visitModule(self
, mod
):
400 static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
402 PyObject *fnames, *result;
405 fnames = PyTuple_New(num_fields);
406 if (!fnames) return NULL;
411 for(i=0; i < num_fields; i++) {
412 PyObject *field = PyString_FromString(fields[i]);
417 PyTuple_SET_ITEM(fnames, i, field);
419 result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
420 type, base, "_fields", fnames, "__module__", "_ast");
422 return (PyTypeObject*)result;
425 static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
428 PyObject *s, *l = PyList_New(num_fields);
430 for(i = 0; i < num_fields; i++) {
431 s = PyString_FromString(attrs[i]);
436 PyList_SET_ITEM(l, i, s);
438 result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0;
443 static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
445 int i, n = asdl_seq_LEN(seq);
446 PyObject *result = PyList_New(n);
450 for (i = 0; i < n; i++) {
451 value = func(asdl_seq_GET(seq, i));
456 PyList_SET_ITEM(result, i, value);
461 static PyObject* ast2obj_object(void *o)
465 Py_INCREF((PyObject*)o);
468 #define ast2obj_identifier ast2obj_object
469 #define ast2obj_string ast2obj_object
470 static PyObject* ast2obj_bool(bool b)
472 return PyBool_FromLong(b);
475 static PyObject* ast2obj_int(long b)
477 return PyInt_FromLong(b);
479 """, 0, reflow
=False)
481 self
.emit("static int init_types(void)",0)
483 self
.emit("static int initialized;", 1)
484 self
.emit("if (initialized) return 1;", 1)
485 self
.emit('AST_type = make_type("AST", &PyBaseObject_Type, NULL, 0);', 1)
488 self
.emit("initialized = 1;", 1)
489 self
.emit("return 1;", 1);
492 def visitProduct(self
, prod
, name
):
494 fields
= name
.value
+"_fields"
497 self
.emit('%s_type = make_type("%s", AST_type, %s, %d);' %
498 (name
, name
, fields
, len(prod
.fields
)), 1)
499 self
.emit("if (!%s_type) return 0;" % name
, 1)
501 def visitSum(self
, sum, name
):
502 self
.emit('%s_type = make_type("%s", AST_type, NULL, 0);' % (name
, name
), 1)
503 self
.emit("if (!%s_type) return 0;" % name
, 1)
505 self
.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
506 (name
, name
, len(sum.attributes
)), 1)
508 self
.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name
, 1)
509 simple
= is_simple(sum)
511 self
.visitConstructor(t
, name
, simple
)
513 def visitConstructor(self
, cons
, name
, simple
):
515 fields
= cons
.name
.value
+"_fields"
518 self
.emit('%s_type = make_type("%s", %s_type, %s, %d);' %
519 (cons
.name
, cons
.name
, name
, fields
, len(cons
.fields
)), 1)
520 self
.emit("if (!%s_type) return 0;" % cons
.name
, 1)
522 self
.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" %
523 (cons
.name
, cons
.name
), 1)
524 self
.emit("if (!%s_singleton) return 0;" % cons
.name
, 1)
526 def parse_version(mod
):
527 return mod
.version
.value
[12:-3]
529 class ASTModuleVisitor(PickleVisitor
):
531 def visitModule(self
, mod
):
532 self
.emit("PyMODINIT_FUNC", 0)
533 self
.emit("init_ast(void)", 0)
535 self
.emit("PyObject *m, *d;", 1)
536 self
.emit("if (!init_types()) return;", 1)
537 self
.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1)
538 self
.emit("if (!m) return;", 1)
539 self
.emit("d = PyModule_GetDict(m);", 1)
540 self
.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return;', 1)
541 self
.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1)
542 self
.emit("return;", 2)
543 # Value of version: "$Revision$"
544 self
.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)'
545 % parse_version(mod
), 1)
546 self
.emit("return;", 2)
551 def visitProduct(self
, prod
, name
):
554 def visitSum(self
, sum, name
):
557 self
.visitConstructor(t
, name
)
559 def visitConstructor(self
, cons
, name
):
560 self
.addObj(cons
.name
)
562 def addObj(self
, name
):
563 self
.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name
, name
), 1)
565 _SPECIALIZED_SEQUENCES
= ('stmt', 'expr')
567 def find_sequence(fields
, doing_specialization
):
568 """Return True if any field uses a sequence."""
571 if not doing_specialization
:
573 if str(f
.type) not in _SPECIALIZED_SEQUENCES
:
577 def has_sequence(types
, doing_specialization
):
579 if find_sequence(t
.fields
, doing_specialization
):
584 class StaticVisitor(PickleVisitor
):
585 CODE
= '''Very simple, always emit this static code. Overide CODE'''
587 def visit(self
, object):
588 self
.emit(self
.CODE
, 0, reflow
=False)
590 class ObjVisitor(PickleVisitor
):
592 def func_begin(self
, name
):
593 ctype
= get_c_type(name
)
594 self
.emit("PyObject*", 0)
595 self
.emit("ast2obj_%s(void* _o)" % (name
), 0)
597 self
.emit("%s o = (%s)_o;" % (ctype
, ctype
), 1)
598 self
.emit("PyObject *result = NULL, *value = NULL;", 1)
599 self
.emit('if (!o) {', 1)
600 self
.emit("Py_INCREF(Py_None);", 2)
601 self
.emit('return Py_None;', 2)
606 self
.emit("return result;", 1)
607 self
.emit("failed:", 0)
608 self
.emit("Py_XDECREF(value);", 1)
609 self
.emit("Py_XDECREF(result);", 1)
610 self
.emit("return NULL;", 1)
614 def visitSum(self
, sum, name
):
616 self
.simpleSum(sum, name
)
618 self
.func_begin(name
)
619 self
.emit("switch (o->kind) {", 1)
620 for i
in range(len(sum.types
)):
622 self
.visitConstructor(t
, i
+ 1, name
)
624 for a
in sum.attributes
:
625 self
.emit("value = ast2obj_%s(o->%s);" % (a
.type, a
.name
), 1)
626 self
.emit("if (!value) goto failed;", 1)
627 self
.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a
.name
, 1)
628 self
.emit('goto failed;', 2)
629 self
.emit('Py_DECREF(value);', 1)
632 def simpleSum(self
, sum, name
):
633 self
.emit("PyObject* ast2obj_%s(%s_ty o)" % (name
, name
), 0)
635 self
.emit("switch(o) {", 1)
637 self
.emit("case %s:" % t
.name
, 2)
638 self
.emit("Py_INCREF(%s_singleton);" % t
.name
, 3)
639 self
.emit("return %s_singleton;" % t
.name
, 3)
641 self
.emit("return NULL; /* cannot happen */", 1)
644 def visitProduct(self
, prod
, name
):
645 self
.func_begin(name
)
646 self
.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name
, 1);
647 self
.emit("if (!result) return NULL;", 1)
648 for field
in prod
.fields
:
649 self
.visitField(field
, name
, 1, True)
652 def visitConstructor(self
, cons
, enum
, name
):
653 self
.emit("case %s_kind:" % cons
.name
, 1)
654 self
.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons
.name
, 2);
655 self
.emit("if (!result) goto failed;", 2)
656 for f
in cons
.fields
:
657 self
.visitField(f
, cons
.name
, 2, False)
658 self
.emit("break;", 2)
660 def visitField(self
, field
, name
, depth
, product
):
662 self
.emit(s
, depth
+ d
)
664 value
= "o->%s" % field
.name
666 value
= "o->v.%s.%s" % (name
, field
.name
)
667 self
.set(field
, value
, depth
)
668 emit("if (!value) goto failed;", 0)
669 emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field
.name
, 0)
670 emit("goto failed;", 1)
671 emit("Py_DECREF(value);", 0)
673 def emitSeq(self
, field
, value
, depth
, emit
):
674 emit("seq = %s;" % value
, 0)
675 emit("n = asdl_seq_LEN(seq);", 0)
676 emit("value = PyList_New(n);", 0)
677 emit("if (!value) goto failed;", 0)
678 emit("for (i = 0; i < n; i++) {", 0)
679 self
.set("value", field
, "asdl_seq_GET(seq, i)", depth
+ 1)
680 emit("if (!value1) goto failed;", 1)
681 emit("PyList_SET_ITEM(value, i, value1);", 1)
682 emit("value1 = NULL;", 1)
685 def set(self
, field
, value
, depth
):
687 # XXX should really check for is_simple, but that requires a symbol table
688 if field
.type.value
== "cmpop":
689 # While the sequence elements are stored as void*,
690 # ast2obj_cmpop expects an enum
691 self
.emit("{", depth
)
692 self
.emit("int i, n = asdl_seq_LEN(%s);" % value
, depth
+1)
693 self
.emit("value = PyList_New(n);", depth
+1)
694 self
.emit("if (!value) goto failed;", depth
+1)
695 self
.emit("for(i = 0; i < n; i++)", depth
+1)
696 # This cannot fail, so no need for error handling
697 self
.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value
,
698 depth
+2, reflow
=False)
699 self
.emit("}", depth
)
701 self
.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value
, field
.type), depth
)
703 ctype
= get_c_type(field
.type)
704 self
.emit("value = ast2obj_%s(%s);" % (field
.type, value
), depth
, reflow
=False)
707 class PartingShots(StaticVisitor
):
710 PyObject* PyAST_mod2obj(mod_ty t)
713 return ast2obj_mod(t);
717 class ChainOfVisitors
:
718 def __init__(self
, *visitors
):
719 self
.visitors
= visitors
721 def visit(self
, object):
722 for v
in self
.visitors
:
726 common_msg
= "/* File automatically generated by %s. */\n"
732 This module must be committed separately after each AST grammar change;
733 The __version__ number is set to the revision number of the commit
734 containing the grammar change.
740 components
= argv0
.split(os
.sep
)
741 argv0
= os
.sep
.join(components
[-2:])
742 auto_gen_msg
= common_msg
% argv0
743 mod
= asdl
.parse(srcfile
)
744 if not asdl
.check(mod
):
747 p
= "%s/%s-ast.h" % (INC_DIR
, mod
.name
)
749 print >> f
, auto_gen_msg
750 print >> f
, '#include "asdl.h"\n'
751 c
= ChainOfVisitors(TypeDefVisitor(f
),
756 print >>f
, "PyObject* PyAST_mod2obj(mod_ty t);"
760 p
= os
.path
.join(SRC_DIR
, str(mod
.name
) + "-ast.c")
762 print >> f
, auto_gen_msg
763 print >> f
, c_file_msg
% parse_version(mod
)
764 print >> f
, '#include "Python.h"'
765 print >> f
, '#include "%s-ast.h"' % mod
.name
767 print >>f
, "static PyTypeObject* AST_type;"
769 PyTypesDeclareVisitor(f
),
779 if __name__
== "__main__":
785 opts
, args
= getopt
.getopt(sys
.argv
[1:], "h:c:")
787 print "Must specify exactly one output file"
795 print "Must specify single input file"