2 """Generate C code from an ASDL description."""
5 # handle fields that have a type but no name
7 import os
, sys
, traceback
15 """Return a string for the C name of the type.
17 This function special cases the default types provided by asdl:
18 identifier, string, int, bool.
20 # XXX ack! need to figure out where Id is useful and where string
21 if isinstance(name
, asdl
.Id
):
23 if name
in asdl
.builtin_types
:
28 def reflow_lines(s
, depth
):
29 """Reflow the line s indented depth tabs.
31 Return a sequence of lines where no line extends beyond MAX_COL
32 when properly indented. The first line is properly indented based
33 exclusively on depth * TABSIZE. All following lines -- these are
34 the reflowed lines generated by this function -- start at the same
35 column as the first character beyond the opening { in the first
38 size
= MAX_COL
- depth
* TABSIZE
45 while len(cur
) > size
:
46 i
= cur
.rfind(' ', 0, size
)
47 # XXX this should be fixed for real
48 if i
== -1 and 'GeneratorExp' in cur
:
50 assert i
!= -1, "Impossible line %d to reflow: %s" % (size
, `s`
)
51 lines
.append(padding
+ cur
[:i
])
53 # find new size based on brace
54 j
= cur
.find('{', 0, i
)
56 j
+= 2 # account for the brace and the space after it
60 j
= cur
.find('(', 0, i
)
62 j
+= 1 # account for the paren (no space after it)
67 lines
.append(padding
+ cur
)
71 """Return True if a sum is a simple.
73 A sum is simple if its types have no fields, e.g.
74 unaryop = Invert | Not | UAdd | USub
82 class EmitVisitor(asdl
.VisitorBase
):
83 """Visit that emits lines"""
85 def __init__(self
, file):
87 super(EmitVisitor
, self
).__init
__()
89 def emit(self
, s
, depth
, reflow
=1):
90 # XXX reflow long lines?
92 lines
= reflow_lines(s
, depth
)
96 line
= (" " * TABSIZE
* depth
) + line
+ "\n"
99 class TypeDefVisitor(EmitVisitor
):
100 def visitModule(self
, mod
):
104 def visitType(self
, type, depth
=0):
105 self
.visit(type.value
, type.name
, depth
)
107 def visitSum(self
, sum, name
, depth
):
109 self
.simple_sum(sum, name
, depth
)
111 self
.sum_with_constructors(sum, name
, depth
)
113 def simple_sum(self
, sum, name
, depth
):
115 for i
in range(len(sum.types
)):
117 enum
.append("%s=%d" % (type.name
, i
+ 1))
118 enums
= ", ".join(enum
)
119 ctype
= get_c_type(name
)
120 s
= "typedef enum _%s { %s } %s;" % (name
, enums
, ctype
)
124 def sum_with_constructors(self
, sum, name
, depth
):
125 ctype
= get_c_type(name
)
126 s
= "typedef struct _%(name)s *%(ctype)s;" % locals()
130 def visitProduct(self
, product
, name
, depth
):
131 ctype
= get_c_type(name
)
132 s
= "typedef struct _%(name)s *%(ctype)s;" % locals()
136 class StructVisitor(EmitVisitor
):
137 """Visitor to generate typdefs for AST."""
139 def visitModule(self
, mod
):
143 def visitType(self
, type, depth
=0):
144 self
.visit(type.value
, type.name
, depth
)
146 def visitSum(self
, sum, name
, depth
):
147 if not is_simple(sum):
148 self
.sum_with_constructors(sum, name
, depth
)
150 def sum_with_constructors(self
, sum, name
, depth
):
151 def emit(s
, depth
=depth
):
152 self
.emit(s
% sys
._getframe
(1).f_locals
, depth
)
154 for i
in range(len(sum.types
)):
156 enum
.append("%s_kind=%d" % (type.name
, i
+ 1))
158 emit("struct _%(name)s {")
159 emit("enum { " + ", ".join(enum
) + " } kind;", depth
+ 1)
160 emit("union {", depth
+ 1)
162 self
.visit(t
, depth
+ 2)
163 emit("} v;", depth
+ 1)
164 for field
in sum.attributes
:
165 # rudimentary attribute handling
166 type = str(field
.type)
167 assert type in asdl
.builtin_types
, type
168 emit("%s %s;" % (type, field
.name
), depth
+ 1);
172 def visitConstructor(self
, cons
, depth
):
174 self
.emit("struct {", depth
)
175 for f
in cons
.fields
:
176 self
.visit(f
, depth
+ 1)
177 self
.emit("} %s;" % cons
.name
, depth
)
180 # XXX not sure what I want here, nothing is probably fine
183 def visitField(self
, field
, depth
):
184 # XXX need to lookup field.type, because it might be something
186 ctype
= get_c_type(field
.type)
189 self
.emit("asdl_seq *%(name)s;" % locals(), depth
)
191 self
.emit("%(ctype)s %(name)s;" % locals(), depth
)
193 def visitProduct(self
, product
, name
, depth
):
194 self
.emit("struct _%(name)s {" % locals(), depth
)
195 for f
in product
.fields
:
196 self
.visit(f
, depth
+ 1)
197 self
.emit("};", depth
)
200 class PrototypeVisitor(EmitVisitor
):
201 """Generate function prototypes for the .h file"""
203 def visitModule(self
, mod
):
207 def visitType(self
, type):
208 self
.visit(type.value
, type.name
)
210 def visitSum(self
, sum, name
):
215 self
.visit(t
, name
, sum.attributes
)
217 def get_args(self
, fields
):
218 """Return list of C argument into, one for each field.
220 Argument info is 3-tuple of a C type, variable name, and flag
221 that is true if type can be NULL.
228 c
= unnamed
[name
] = unnamed
.get(name
, 0) + 1
230 name
= "name%d" % (c
- 1)
233 # XXX should extend get_c_type() to handle this
237 ctype
= get_c_type(f
.type)
238 args
.append((ctype
, name
, f
.opt
or f
.seq
))
241 def visitConstructor(self
, cons
, type, attrs
):
242 args
= self
.get_args(cons
.fields
)
243 attrs
= self
.get_args(attrs
)
244 ctype
= get_c_type(type)
245 self
.emit_function(cons
.name
, ctype
, args
, attrs
)
247 def emit_function(self
, name
, ctype
, args
, attrs
, union
=1):
250 argstr
= ", ".join(["%s %s" % (atype
, aname
)
251 for atype
, aname
, opt
in args
])
252 argstr
+= ", PyArena *arena"
254 argstr
= "PyArena *arena"
255 self
.emit("%s %s(%s);" % (ctype
, name
, argstr
), 0)
257 def visitProduct(self
, prod
, name
):
258 self
.emit_function(name
, get_c_type(name
),
259 self
.get_args(prod
.fields
), [], union
=0)
261 class FunctionVisitor(PrototypeVisitor
):
262 """Visitor to generate constructor functions for AST."""
264 def emit_function(self
, name
, ctype
, args
, attrs
, union
=1):
265 def emit(s
, depth
=0, reflow
=1):
266 self
.emit(s
, depth
, reflow
)
267 argstr
= ", ".join(["%s %s" % (atype
, aname
)
268 for atype
, aname
, opt
in args
+ attrs
])
270 argstr
+= ", PyArena *arena"
272 argstr
= "PyArena *arena"
273 self
.emit("%s" % ctype
, 0)
274 emit("%s(%s)" % (name
, argstr
))
276 emit("%s p;" % ctype
, 1)
277 for argtype
, argname
, opt
in args
:
278 # XXX hack alert: false is allowed for a bool
279 if not opt
and not argtype
== "bool":
280 emit("if (!%s) {" % argname
, 1)
281 emit("PyErr_SetString(PyExc_ValueError,", 2)
282 msg
= "field %s is required for %s" % (argname
, name
)
283 emit(' "%s");' % msg
,
285 emit('return NULL;', 2)
288 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype
, 1);
290 emit("PyErr_NoMemory();", 2)
291 emit("return NULL;", 2)
294 self
.emit_body_union(name
, args
, attrs
)
296 self
.emit_body_struct(name
, args
, attrs
)
301 def emit_body_union(self
, name
, args
, attrs
):
302 def emit(s
, depth
=0, reflow
=1):
303 self
.emit(s
, depth
, reflow
)
304 emit("p->kind = %s_kind;" % name
, 1)
305 for argtype
, argname
, opt
in args
:
306 emit("p->v.%s.%s = %s;" % (name
, argname
, argname
), 1)
307 for argtype
, argname
, opt
in attrs
:
308 emit("p->%s = %s;" % (argname
, argname
), 1)
310 def emit_body_struct(self
, name
, args
, attrs
):
311 def emit(s
, depth
=0, reflow
=1):
312 self
.emit(s
, depth
, reflow
)
313 for argtype
, argname
, opt
in args
:
314 emit("p->%s = %s;" % (argname
, argname
), 1)
317 class PickleVisitor(EmitVisitor
):
319 def visitModule(self
, mod
):
323 def visitType(self
, type):
324 self
.visit(type.value
, type.name
)
326 def visitSum(self
, sum, name
):
329 def visitProduct(self
, sum, name
):
332 def visitConstructor(self
, cons
, name
):
335 def visitField(self
, sum):
338 class MarshalPrototypeVisitor(PickleVisitor
):
340 def prototype(self
, sum, name
):
341 ctype
= get_c_type(name
)
342 self
.emit("static int marshal_write_%s(PyObject **, int *, %s);"
345 visitProduct
= visitSum
= prototype
347 class FreePrototypeVisitor(PickleVisitor
):
349 def prototype(self
, sum, name
):
350 ctype
= get_c_type(name
)
351 self
.emit("void free_%s(%s);" % (name
, ctype
), 0)
353 visitProduct
= visitSum
= prototype
355 _SPECIALIZED_SEQUENCES
= ('stmt', 'expr')
357 def find_sequence(fields
, doing_specialization
):
358 """Return True if any field uses a sequence."""
361 if not doing_specialization
:
363 if str(f
.type) not in _SPECIALIZED_SEQUENCES
:
367 def has_sequence(types
, doing_specialization
):
369 if find_sequence(t
.fields
, doing_specialization
):
374 class StaticVisitor(PickleVisitor
):
375 CODE
= '''Very simple, always emit this static code. Overide CODE'''
377 def visit(self
, object):
378 self
.emit(self
.CODE
, 0, reflow
=False)
380 class FreeUtilVisitor(StaticVisitor
):
382 CODE
= '''static void
383 free_seq_exprs(asdl_seq *seq)
386 n = asdl_seq_LEN(seq);
387 for (i = 0; i < n; i++)
388 free_expr((expr_ty)asdl_seq_GET(seq, i));
393 free_seq_stmts(asdl_seq *seq)
396 n = asdl_seq_LEN(seq);
397 for (i = 0; i < n; i++)
398 free_stmt((stmt_ty)asdl_seq_GET(seq, i));
403 class FreeVisitor(PickleVisitor
):
405 def func_begin(self
, name
, has_seq
):
406 ctype
= get_c_type(name
)
408 self
.emit("free_%s(%s o)" % (name
, ctype
), 0)
411 self
.emit("int i, n;", 1)
412 self
.emit("asdl_seq *seq;", 1)
414 self
.emit('if (!o)', 1)
415 self
.emit('return;', 2)
422 def visitSum(self
, sum, name
):
423 has_seq
= has_sequence(sum.types
, True)
424 self
.func_begin(name
, has_seq
)
425 if not is_simple(sum):
426 self
.emit("switch (o->kind) {", 1)
427 for i
in range(len(sum.types
)):
429 self
.visitConstructor(t
, i
+ 1, name
)
432 self
.emit("free(o);", 1)
435 def visitProduct(self
, prod
, name
):
436 self
.func_begin(name
, find_sequence(prod
.fields
, True))
437 for field
in prod
.fields
:
438 self
.visitField(field
, name
, 1, True)
440 self
.emit("free(o);", 1)
443 def visitConstructor(self
, cons
, enum
, name
):
444 self
.emit("case %s_kind:" % cons
.name
, 1)
445 for f
in cons
.fields
:
446 self
.visitField(f
, cons
.name
, 2, False)
447 self
.emit("break;", 2)
449 def visitField(self
, field
, name
, depth
, product
):
451 self
.emit(s
, depth
+ d
)
453 value
= "o->%s" % field
.name
455 value
= "o->v.%s.%s" % (name
, field
.name
)
457 self
.emitSeq(field
, value
, depth
, emit
)
459 # XXX need to know the simple types in advance, so that we
460 # don't call free_TYPE() for them.
463 emit("if (%s) {" % value
, 0)
464 self
.free(field
, value
, depth
+ 1)
467 self
.free(field
, value
, depth
)
469 def emitSeq(self
, field
, value
, depth
, emit
):
470 # specialize for freeing sequences of statements and expressions
471 if str(field
.type) in _SPECIALIZED_SEQUENCES
:
472 c_code
= "free_seq_%ss(%s);" % (field
.type, value
)
475 emit("seq = %s;" % value
, 0)
476 emit("n = asdl_seq_LEN(seq);", 0)
477 emit("for (i = 0; i < n; i++)", 0)
478 self
.free(field
, "asdl_seq_GET(seq, i)", depth
+ 1)
479 emit("asdl_seq_free(seq);", 0)
481 def free(self
, field
, value
, depth
):
482 if str(field
.type) in ("identifier", "string", "object"):
483 ctype
= get_c_type(field
.type)
484 self
.emit("Py_DECREF((%s)%s);" % (ctype
, value
), depth
)
485 elif str(field
.type) == "bool":
488 ctype
= get_c_type(field
.type)
489 self
.emit("free_%s((%s)%s);" % (field
.type, ctype
, value
), depth
)
492 class MarshalUtilVisitor(StaticVisitor
):
495 #define CHECKSIZE(BUF, OFF, MIN) { \\
496 int need = *(OFF) + MIN; \\
497 if (need >= PyString_GET_SIZE(*(BUF))) { \\
498 int newsize = PyString_GET_SIZE(*(BUF)) * 2; \\
499 if (newsize < need) \\
501 if (_PyString_Resize((BUF), newsize) < 0) \\
507 marshal_write_int(PyObject **buf, int *offset, int x)
511 CHECKSIZE(buf, offset, 4)
512 s = PyString_AS_STRING(*buf) + (*offset);
514 s[1] = (x >> 8) & 0xff;
515 s[2] = (x >> 16) & 0xff;
516 s[3] = (x >> 24) & 0xff;
522 marshal_write_bool(PyObject **buf, int *offset, bool b)
525 marshal_write_int(buf, offset, 1);
527 marshal_write_int(buf, offset, 0);
532 marshal_write_identifier(PyObject **buf, int *offset, identifier id)
534 int l = PyString_GET_SIZE(id);
535 marshal_write_int(buf, offset, l);
536 CHECKSIZE(buf, offset, l);
537 memcpy(PyString_AS_STRING(*buf) + *offset,
538 PyString_AS_STRING(id), l);
544 marshal_write_string(PyObject **buf, int *offset, string s)
546 int len = PyString_GET_SIZE(s);
547 marshal_write_int(buf, offset, len);
548 CHECKSIZE(buf, offset, len);
549 memcpy(PyString_AS_STRING(*buf) + *offset,
550 PyString_AS_STRING(s), len);
556 marshal_write_object(PyObject **buf, int *offset, object s)
563 class MarshalFunctionVisitor(PickleVisitor
):
565 def func_begin(self
, name
, has_seq
):
566 ctype
= get_c_type(name
)
567 self
.emit("static int", 0)
568 self
.emit("marshal_write_%s(PyObject **buf, int *off, %s o)" %
572 self
.emit("int i;", 1)
575 self
.emit("return 1;", 1)
579 def visitSum(self
, sum, name
):
580 self
.func_begin(name
, has_sequence(sum.types
, False))
581 simple
= is_simple(sum)
583 self
.emit("switch (o) {", 1)
585 self
.emit("switch (o->kind) {", 1)
586 for i
in range(len(sum.types
)):
588 self
.visitConstructor(t
, i
+ 1, name
, simple
)
592 def visitProduct(self
, prod
, name
):
593 self
.func_begin(name
, find_sequence(prod
.fields
, False))
594 for field
in prod
.fields
:
595 self
.visitField(field
, name
, 1, 1)
598 def visitConstructor(self
, cons
, enum
, name
, simple
):
600 self
.emit("case %s:" % cons
.name
, 1)
601 self
.emit("marshal_write_int(buf, off, %d);" % enum
, 2);
602 self
.emit("break;", 2)
604 self
.emit("case %s_kind:" % cons
.name
, 1)
605 self
.emit("marshal_write_int(buf, off, %d);" % enum
, 2)
606 for f
in cons
.fields
:
607 self
.visitField(f
, cons
.name
, 2, 0)
608 self
.emit("break;", 2)
610 def visitField(self
, field
, name
, depth
, product
):
612 self
.emit(s
, depth
+ d
)
614 value
= "o->%s" % field
.name
616 value
= "o->v.%s.%s" % (name
, field
.name
)
618 emit("marshal_write_int(buf, off, asdl_seq_LEN(%s));" % value
, 0)
619 emit("for (i = 0; i < asdl_seq_LEN(%s); i++) {" % value
, 0)
620 emit("void *elt = asdl_seq_GET(%s, i);" % value
, 1);
621 ctype
= get_c_type(field
.type);
622 emit("marshal_write_%s(buf, off, (%s)elt);" % (field
.type,
626 emit("if (%s) {" % value
, 0)
627 emit("marshal_write_int(buf, off, 1);", 1)
628 emit("marshal_write_%s(buf, off, %s);" % (field
.type, value
), 1)
631 emit("marshal_write_int(buf, off, 0);", 1)
634 emit("marshal_write_%s(buf, off, %s);" % (field
.type, value
), 0)
636 class ChainOfVisitors
:
637 def __init__(self
, *visitors
):
638 self
.visitors
= visitors
640 def visit(self
, object):
641 for v
in self
.visitors
:
647 components
= argv0
.split(os
.sep
)
648 argv0
= os
.sep
.join(components
[-2:])
649 auto_gen_msg
= '/* File automatically generated by %s */\n' % argv0
650 mod
= asdl
.parse(srcfile
)
651 if not asdl
.check(mod
):
654 p
= "%s/%s-ast.h" % (INC_DIR
, mod
.name
)
656 p
= "%s-ast.h" % mod
.name
658 print >> f
, auto_gen_msg
659 print >> f
, '#include "asdl.h"\n'
660 c
= ChainOfVisitors(TypeDefVisitor(f
),
663 ## FreePrototypeVisitor(f),
669 p
= "%s/%s-ast.c" % (SRC_DIR
, mod
.name
)
671 p
= "%s-ast.c" % mod
.name
673 print >> f
, auto_gen_msg
674 print >> f
, '#include "Python.h"'
675 print >> f
, '#include "%s-ast.h"' % mod
.name
677 v
= ChainOfVisitors(MarshalPrototypeVisitor(f
),
679 ## FreeUtilVisitor(f),
681 MarshalUtilVisitor(f
),
682 MarshalFunctionVisitor(f
),
687 if __name__
== "__main__":
693 opts
, args
= getopt
.getopt(sys
.argv
[1:], "h:c:")
700 print "Must specify single input file"