Exceptions raised during renaming in rotating file handlers are now passed to handleE...
[python.git] / Parser / asdl_c.py
blob766be67a07cfec881b6b1b553481bf236af1237a
1 #! /usr/bin/env python
2 """Generate C code from an ASDL description."""
4 # TO DO
5 # handle fields that have a type but no name
7 import os, sys, traceback
9 import asdl
11 TABSIZE = 8
12 MAX_COL = 80
14 def get_c_type(name):
15 """Return a string for the C name of the type.
17 This function special cases the default types provided by asdl:
18 identifier, string, int, bool.
19 """
20 # XXX ack! need to figure out where Id is useful and where string
21 if isinstance(name, asdl.Id):
22 name = name.value
23 if name in asdl.builtin_types:
24 return name
25 else:
26 return "%s_ty" % name
28 def reflow_lines(s, depth):
29 """Reflow the line s indented depth tabs.
31 Return a sequence of lines where no line extends beyond MAX_COL
32 when properly indented. The first line is properly indented based
33 exclusively on depth * TABSIZE. All following lines -- these are
34 the reflowed lines generated by this function -- start at the same
35 column as the first character beyond the opening { in the first
36 line.
37 """
38 size = MAX_COL - depth * TABSIZE
39 if len(s) < size:
40 return [s]
42 lines = []
43 cur = s
44 padding = ""
45 while len(cur) > size:
46 i = cur.rfind(' ', 0, size)
47 # XXX this should be fixed for real
48 if i == -1 and 'GeneratorExp' in cur:
49 i = size + 3
50 assert i != -1, "Impossible line %d to reflow: %s" % (size, `s`)
51 lines.append(padding + cur[:i])
52 if len(lines) == 1:
53 # find new size based on brace
54 j = cur.find('{', 0, i)
55 if j >= 0:
56 j += 2 # account for the brace and the space after it
57 size -= j
58 padding = " " * j
59 else:
60 j = cur.find('(', 0, i)
61 if j >= 0:
62 j += 1 # account for the paren (no space after it)
63 size -= j
64 padding = " " * j
65 cur = cur[i+1:]
66 else:
67 lines.append(padding + cur)
68 return lines
70 def is_simple(sum):
71 """Return True if a sum is a simple.
73 A sum is simple if its types have no fields, e.g.
74 unaryop = Invert | Not | UAdd | USub
75 """
77 for t in sum.types:
78 if t.fields:
79 return False
80 return True
82 class EmitVisitor(asdl.VisitorBase):
83 """Visit that emits lines"""
85 def __init__(self, file):
86 self.file = file
87 super(EmitVisitor, self).__init__()
89 def emit(self, s, depth, reflow=1):
90 # XXX reflow long lines?
91 if reflow:
92 lines = reflow_lines(s, depth)
93 else:
94 lines = [s]
95 for line in lines:
96 line = (" " * TABSIZE * depth) + line + "\n"
97 self.file.write(line)
99 class TypeDefVisitor(EmitVisitor):
100 def visitModule(self, mod):
101 for dfn in mod.dfns:
102 self.visit(dfn)
104 def visitType(self, type, depth=0):
105 self.visit(type.value, type.name, depth)
107 def visitSum(self, sum, name, depth):
108 if is_simple(sum):
109 self.simple_sum(sum, name, depth)
110 else:
111 self.sum_with_constructors(sum, name, depth)
113 def simple_sum(self, sum, name, depth):
114 enum = []
115 for i in range(len(sum.types)):
116 type = sum.types[i]
117 enum.append("%s=%d" % (type.name, i + 1))
118 enums = ", ".join(enum)
119 ctype = get_c_type(name)
120 s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
121 self.emit(s, depth)
122 self.emit("", depth)
124 def sum_with_constructors(self, sum, name, depth):
125 ctype = get_c_type(name)
126 s = "typedef struct _%(name)s *%(ctype)s;" % locals()
127 self.emit(s, depth)
128 self.emit("", depth)
130 def visitProduct(self, product, name, depth):
131 ctype = get_c_type(name)
132 s = "typedef struct _%(name)s *%(ctype)s;" % locals()
133 self.emit(s, depth)
134 self.emit("", depth)
136 class StructVisitor(EmitVisitor):
137 """Visitor to generate typdefs for AST."""
139 def visitModule(self, mod):
140 for dfn in mod.dfns:
141 self.visit(dfn)
143 def visitType(self, type, depth=0):
144 self.visit(type.value, type.name, depth)
146 def visitSum(self, sum, name, depth):
147 if not is_simple(sum):
148 self.sum_with_constructors(sum, name, depth)
150 def sum_with_constructors(self, sum, name, depth):
151 def emit(s, depth=depth):
152 self.emit(s % sys._getframe(1).f_locals, depth)
153 enum = []
154 for i in range(len(sum.types)):
155 type = sum.types[i]
156 enum.append("%s_kind=%d" % (type.name, i + 1))
158 emit("struct _%(name)s {")
159 emit("enum { " + ", ".join(enum) + " } kind;", depth + 1)
160 emit("union {", depth + 1)
161 for t in sum.types:
162 self.visit(t, depth + 2)
163 emit("} v;", depth + 1)
164 for field in sum.attributes:
165 # rudimentary attribute handling
166 type = str(field.type)
167 assert type in asdl.builtin_types, type
168 emit("%s %s;" % (type, field.name), depth + 1);
169 emit("};")
170 emit("")
172 def visitConstructor(self, cons, depth):
173 if cons.fields:
174 self.emit("struct {", depth)
175 for f in cons.fields:
176 self.visit(f, depth + 1)
177 self.emit("} %s;" % cons.name, depth)
178 self.emit("", depth)
179 else:
180 # XXX not sure what I want here, nothing is probably fine
181 pass
183 def visitField(self, field, depth):
184 # XXX need to lookup field.type, because it might be something
185 # like a builtin...
186 ctype = get_c_type(field.type)
187 name = field.name
188 if field.seq:
189 self.emit("asdl_seq *%(name)s;" % locals(), depth)
190 else:
191 self.emit("%(ctype)s %(name)s;" % locals(), depth)
193 def visitProduct(self, product, name, depth):
194 self.emit("struct _%(name)s {" % locals(), depth)
195 for f in product.fields:
196 self.visit(f, depth + 1)
197 self.emit("};", depth)
198 self.emit("", depth)
200 class PrototypeVisitor(EmitVisitor):
201 """Generate function prototypes for the .h file"""
203 def visitModule(self, mod):
204 for dfn in mod.dfns:
205 self.visit(dfn)
207 def visitType(self, type):
208 self.visit(type.value, type.name)
210 def visitSum(self, sum, name):
211 if is_simple(sum):
212 pass # XXX
213 else:
214 for t in sum.types:
215 self.visit(t, name, sum.attributes)
217 def get_args(self, fields):
218 """Return list of C argument into, one for each field.
220 Argument info is 3-tuple of a C type, variable name, and flag
221 that is true if type can be NULL.
223 args = []
224 unnamed = {}
225 for f in fields:
226 if f.name is None:
227 name = f.type
228 c = unnamed[name] = unnamed.get(name, 0) + 1
229 if c > 1:
230 name = "name%d" % (c - 1)
231 else:
232 name = f.name
233 # XXX should extend get_c_type() to handle this
234 if f.seq:
235 ctype = "asdl_seq *"
236 else:
237 ctype = get_c_type(f.type)
238 args.append((ctype, name, f.opt or f.seq))
239 return args
241 def visitConstructor(self, cons, type, attrs):
242 args = self.get_args(cons.fields)
243 attrs = self.get_args(attrs)
244 ctype = get_c_type(type)
245 self.emit_function(cons.name, ctype, args, attrs)
247 def emit_function(self, name, ctype, args, attrs, union=1):
248 args = args + attrs
249 if args:
250 argstr = ", ".join(["%s %s" % (atype, aname)
251 for atype, aname, opt in args])
252 argstr += ", PyArena *arena"
253 else:
254 argstr = "PyArena *arena"
255 self.emit("%s %s(%s);" % (ctype, name, argstr), 0)
257 def visitProduct(self, prod, name):
258 self.emit_function(name, get_c_type(name),
259 self.get_args(prod.fields), [], union=0)
261 class FunctionVisitor(PrototypeVisitor):
262 """Visitor to generate constructor functions for AST."""
264 def emit_function(self, name, ctype, args, attrs, union=1):
265 def emit(s, depth=0, reflow=1):
266 self.emit(s, depth, reflow)
267 argstr = ", ".join(["%s %s" % (atype, aname)
268 for atype, aname, opt in args + attrs])
269 if argstr:
270 argstr += ", PyArena *arena"
271 else:
272 argstr = "PyArena *arena"
273 self.emit("%s" % ctype, 0)
274 emit("%s(%s)" % (name, argstr))
275 emit("{")
276 emit("%s p;" % ctype, 1)
277 for argtype, argname, opt in args:
278 # XXX hack alert: false is allowed for a bool
279 if not opt and not argtype == "bool":
280 emit("if (!%s) {" % argname, 1)
281 emit("PyErr_SetString(PyExc_ValueError,", 2)
282 msg = "field %s is required for %s" % (argname, name)
283 emit(' "%s");' % msg,
284 2, reflow=0)
285 emit('return NULL;', 2)
286 emit('}', 1)
288 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
289 emit("if (!p) {", 1)
290 emit("PyErr_NoMemory();", 2)
291 emit("return NULL;", 2)
292 emit("}", 1)
293 if union:
294 self.emit_body_union(name, args, attrs)
295 else:
296 self.emit_body_struct(name, args, attrs)
297 emit("return p;", 1)
298 emit("}")
299 emit("")
301 def emit_body_union(self, name, args, attrs):
302 def emit(s, depth=0, reflow=1):
303 self.emit(s, depth, reflow)
304 emit("p->kind = %s_kind;" % name, 1)
305 for argtype, argname, opt in args:
306 emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
307 for argtype, argname, opt in attrs:
308 emit("p->%s = %s;" % (argname, argname), 1)
310 def emit_body_struct(self, name, args, attrs):
311 def emit(s, depth=0, reflow=1):
312 self.emit(s, depth, reflow)
313 for argtype, argname, opt in args:
314 emit("p->%s = %s;" % (argname, argname), 1)
315 assert not attrs
317 class PickleVisitor(EmitVisitor):
319 def visitModule(self, mod):
320 for dfn in mod.dfns:
321 self.visit(dfn)
323 def visitType(self, type):
324 self.visit(type.value, type.name)
326 def visitSum(self, sum, name):
327 pass
329 def visitProduct(self, sum, name):
330 pass
332 def visitConstructor(self, cons, name):
333 pass
335 def visitField(self, sum):
336 pass
338 class MarshalPrototypeVisitor(PickleVisitor):
340 def prototype(self, sum, name):
341 ctype = get_c_type(name)
342 self.emit("static int marshal_write_%s(PyObject **, int *, %s);"
343 % (name, ctype), 0)
345 visitProduct = visitSum = prototype
347 class FreePrototypeVisitor(PickleVisitor):
349 def prototype(self, sum, name):
350 ctype = get_c_type(name)
351 self.emit("void free_%s(%s);" % (name, ctype), 0)
353 visitProduct = visitSum = prototype
355 _SPECIALIZED_SEQUENCES = ('stmt', 'expr')
357 def find_sequence(fields, doing_specialization):
358 """Return True if any field uses a sequence."""
359 for f in fields:
360 if f.seq:
361 if not doing_specialization:
362 return True
363 if str(f.type) not in _SPECIALIZED_SEQUENCES:
364 return True
365 return False
367 def has_sequence(types, doing_specialization):
368 for t in types:
369 if find_sequence(t.fields, doing_specialization):
370 return True
371 return False
374 class StaticVisitor(PickleVisitor):
375 CODE = '''Very simple, always emit this static code. Overide CODE'''
377 def visit(self, object):
378 self.emit(self.CODE, 0, reflow=False)
380 class FreeUtilVisitor(StaticVisitor):
382 CODE = '''static void
383 free_seq_exprs(asdl_seq *seq)
385 int i, n;
386 n = asdl_seq_LEN(seq);
387 for (i = 0; i < n; i++)
388 free_expr((expr_ty)asdl_seq_GET(seq, i));
389 asdl_seq_free(seq);
392 static void
393 free_seq_stmts(asdl_seq *seq)
395 int i, n;
396 n = asdl_seq_LEN(seq);
397 for (i = 0; i < n; i++)
398 free_stmt((stmt_ty)asdl_seq_GET(seq, i));
399 asdl_seq_free(seq);
403 class FreeVisitor(PickleVisitor):
405 def func_begin(self, name, has_seq):
406 ctype = get_c_type(name)
407 self.emit("void", 0)
408 self.emit("free_%s(%s o)" % (name, ctype), 0)
409 self.emit("{", 0)
410 if has_seq:
411 self.emit("int i, n;", 1)
412 self.emit("asdl_seq *seq;", 1)
413 self.emit('', 0)
414 self.emit('if (!o)', 1)
415 self.emit('return;', 2)
416 self.emit('', 0)
418 def func_end(self):
419 self.emit("}", 0)
420 self.emit("", 0)
422 def visitSum(self, sum, name):
423 has_seq = has_sequence(sum.types, True)
424 self.func_begin(name, has_seq)
425 if not is_simple(sum):
426 self.emit("switch (o->kind) {", 1)
427 for i in range(len(sum.types)):
428 t = sum.types[i]
429 self.visitConstructor(t, i + 1, name)
430 self.emit("}", 1)
431 self.emit("", 0)
432 self.emit("free(o);", 1)
433 self.func_end()
435 def visitProduct(self, prod, name):
436 self.func_begin(name, find_sequence(prod.fields, True))
437 for field in prod.fields:
438 self.visitField(field, name, 1, True)
439 self.emit("", 0)
440 self.emit("free(o);", 1)
441 self.func_end()
443 def visitConstructor(self, cons, enum, name):
444 self.emit("case %s_kind:" % cons.name, 1)
445 for f in cons.fields:
446 self.visitField(f, cons.name, 2, False)
447 self.emit("break;", 2)
449 def visitField(self, field, name, depth, product):
450 def emit(s, d):
451 self.emit(s, depth + d)
452 if product:
453 value = "o->%s" % field.name
454 else:
455 value = "o->v.%s.%s" % (name, field.name)
456 if field.seq:
457 self.emitSeq(field, value, depth, emit)
459 # XXX need to know the simple types in advance, so that we
460 # don't call free_TYPE() for them.
462 elif field.opt:
463 emit("if (%s) {" % value, 0)
464 self.free(field, value, depth + 1)
465 emit("}", 0)
466 else:
467 self.free(field, value, depth)
469 def emitSeq(self, field, value, depth, emit):
470 # specialize for freeing sequences of statements and expressions
471 if str(field.type) in _SPECIALIZED_SEQUENCES:
472 c_code = "free_seq_%ss(%s);" % (field.type, value)
473 emit(c_code, 0)
474 else:
475 emit("seq = %s;" % value, 0)
476 emit("n = asdl_seq_LEN(seq);", 0)
477 emit("for (i = 0; i < n; i++)", 0)
478 self.free(field, "asdl_seq_GET(seq, i)", depth + 1)
479 emit("asdl_seq_free(seq);", 0)
481 def free(self, field, value, depth):
482 if str(field.type) in ("identifier", "string", "object"):
483 ctype = get_c_type(field.type)
484 self.emit("Py_DECREF((%s)%s);" % (ctype, value), depth)
485 elif str(field.type) == "bool":
486 return
487 else:
488 ctype = get_c_type(field.type)
489 self.emit("free_%s((%s)%s);" % (field.type, ctype, value), depth)
492 class MarshalUtilVisitor(StaticVisitor):
494 CODE = '''
495 #define CHECKSIZE(BUF, OFF, MIN) { \\
496 int need = *(OFF) + MIN; \\
497 if (need >= PyString_GET_SIZE(*(BUF))) { \\
498 int newsize = PyString_GET_SIZE(*(BUF)) * 2; \\
499 if (newsize < need) \\
500 newsize = need; \\
501 if (_PyString_Resize((BUF), newsize) < 0) \\
502 return 0; \\
503 } \\
506 static int
507 marshal_write_int(PyObject **buf, int *offset, int x)
509 char *s;
511 CHECKSIZE(buf, offset, 4)
512 s = PyString_AS_STRING(*buf) + (*offset);
513 s[0] = (x & 0xff);
514 s[1] = (x >> 8) & 0xff;
515 s[2] = (x >> 16) & 0xff;
516 s[3] = (x >> 24) & 0xff;
517 *offset += 4;
518 return 1;
521 static int
522 marshal_write_bool(PyObject **buf, int *offset, bool b)
524 if (b)
525 marshal_write_int(buf, offset, 1);
526 else
527 marshal_write_int(buf, offset, 0);
528 return 1;
531 static int
532 marshal_write_identifier(PyObject **buf, int *offset, identifier id)
534 int l = PyString_GET_SIZE(id);
535 marshal_write_int(buf, offset, l);
536 CHECKSIZE(buf, offset, l);
537 memcpy(PyString_AS_STRING(*buf) + *offset,
538 PyString_AS_STRING(id), l);
539 *offset += l;
540 return 1;
543 static int
544 marshal_write_string(PyObject **buf, int *offset, string s)
546 int len = PyString_GET_SIZE(s);
547 marshal_write_int(buf, offset, len);
548 CHECKSIZE(buf, offset, len);
549 memcpy(PyString_AS_STRING(*buf) + *offset,
550 PyString_AS_STRING(s), len);
551 *offset += len;
552 return 1;
555 static int
556 marshal_write_object(PyObject **buf, int *offset, object s)
558 /* XXX */
559 return 0;
563 class MarshalFunctionVisitor(PickleVisitor):
565 def func_begin(self, name, has_seq):
566 ctype = get_c_type(name)
567 self.emit("static int", 0)
568 self.emit("marshal_write_%s(PyObject **buf, int *off, %s o)" %
569 (name, ctype), 0)
570 self.emit("{", 0)
571 if has_seq:
572 self.emit("int i;", 1)
574 def func_end(self):
575 self.emit("return 1;", 1)
576 self.emit("}", 0)
577 self.emit("", 0)
579 def visitSum(self, sum, name):
580 self.func_begin(name, has_sequence(sum.types, False))
581 simple = is_simple(sum)
582 if simple:
583 self.emit("switch (o) {", 1)
584 else:
585 self.emit("switch (o->kind) {", 1)
586 for i in range(len(sum.types)):
587 t = sum.types[i]
588 self.visitConstructor(t, i + 1, name, simple)
589 self.emit("}", 1)
590 self.func_end()
592 def visitProduct(self, prod, name):
593 self.func_begin(name, find_sequence(prod.fields, False))
594 for field in prod.fields:
595 self.visitField(field, name, 1, 1)
596 self.func_end()
598 def visitConstructor(self, cons, enum, name, simple):
599 if simple:
600 self.emit("case %s:" % cons.name, 1)
601 self.emit("marshal_write_int(buf, off, %d);" % enum, 2);
602 self.emit("break;", 2)
603 else:
604 self.emit("case %s_kind:" % cons.name, 1)
605 self.emit("marshal_write_int(buf, off, %d);" % enum, 2)
606 for f in cons.fields:
607 self.visitField(f, cons.name, 2, 0)
608 self.emit("break;", 2)
610 def visitField(self, field, name, depth, product):
611 def emit(s, d):
612 self.emit(s, depth + d)
613 if product:
614 value = "o->%s" % field.name
615 else:
616 value = "o->v.%s.%s" % (name, field.name)
617 if field.seq:
618 emit("marshal_write_int(buf, off, asdl_seq_LEN(%s));" % value, 0)
619 emit("for (i = 0; i < asdl_seq_LEN(%s); i++) {" % value, 0)
620 emit("void *elt = asdl_seq_GET(%s, i);" % value, 1);
621 ctype = get_c_type(field.type);
622 emit("marshal_write_%s(buf, off, (%s)elt);" % (field.type,
623 ctype), 1)
624 emit("}", 0)
625 elif field.opt:
626 emit("if (%s) {" % value, 0)
627 emit("marshal_write_int(buf, off, 1);", 1)
628 emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 1)
629 emit("}", 0)
630 emit("else {", 0)
631 emit("marshal_write_int(buf, off, 0);", 1)
632 emit("}", 0)
633 else:
634 emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 0)
636 class ChainOfVisitors:
637 def __init__(self, *visitors):
638 self.visitors = visitors
640 def visit(self, object):
641 for v in self.visitors:
642 v.visit(object)
643 v.emit("", 0)
645 def main(srcfile):
646 argv0 = sys.argv[0]
647 components = argv0.split(os.sep)
648 argv0 = os.sep.join(components[-2:])
649 auto_gen_msg = '/* File automatically generated by %s */\n' % argv0
650 mod = asdl.parse(srcfile)
651 if not asdl.check(mod):
652 sys.exit(1)
653 if INC_DIR:
654 p = "%s/%s-ast.h" % (INC_DIR, mod.name)
655 else:
656 p = "%s-ast.h" % mod.name
657 f = open(p, "wb")
658 print >> f, auto_gen_msg
659 print >> f, '#include "asdl.h"\n'
660 c = ChainOfVisitors(TypeDefVisitor(f),
661 StructVisitor(f),
662 PrototypeVisitor(f),
663 ## FreePrototypeVisitor(f),
665 c.visit(mod)
666 f.close()
668 if SRC_DIR:
669 p = "%s/%s-ast.c" % (SRC_DIR, mod.name)
670 else:
671 p = "%s-ast.c" % mod.name
672 f = open(p, "wb")
673 print >> f, auto_gen_msg
674 print >> f, '#include "Python.h"'
675 print >> f, '#include "%s-ast.h"' % mod.name
676 print >> f
677 v = ChainOfVisitors(MarshalPrototypeVisitor(f),
678 FunctionVisitor(f),
679 ## FreeUtilVisitor(f),
680 ## FreeVisitor(f),
681 MarshalUtilVisitor(f),
682 MarshalFunctionVisitor(f),
684 v.visit(mod)
685 f.close()
687 if __name__ == "__main__":
688 import sys
689 import getopt
691 INC_DIR = ''
692 SRC_DIR = ''
693 opts, args = getopt.getopt(sys.argv[1:], "h:c:")
694 for o, v in opts:
695 if o == '-h':
696 INC_DIR = v
697 if o == '-c':
698 SRC_DIR = v
699 if len(args) != 1:
700 print "Must specify single input file"
701 main(args[0])