Issue #7318: multiprocessing now uses a timeout when it fails to establish
[python.git] / Lib / compiler / pycodegen.py
blob81234d1ff00a7b02b894e87e499d3683bf742b08
1 import imp
2 import os
3 import marshal
4 import struct
5 import sys
6 from cStringIO import StringIO
8 from compiler import ast, parse, walk, syntax
9 from compiler import pyassem, misc, future, symbols
10 from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \
11 SC_FREE, SC_CELL
12 from compiler.consts import (CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,
13 CO_NESTED, CO_GENERATOR, CO_FUTURE_DIVISION,
14 CO_FUTURE_ABSIMPORT, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_PRINT_FUNCTION)
15 from compiler.pyassem import TupleArg
17 # XXX The version-specific code can go, since this code only works with 2.x.
18 # Do we have Python 1.x or Python 2.x?
19 try:
20 VERSION = sys.version_info[0]
21 except AttributeError:
22 VERSION = 1
24 callfunc_opcode_info = {
25 # (Have *args, Have **args) : opcode
26 (0,0) : "CALL_FUNCTION",
27 (1,0) : "CALL_FUNCTION_VAR",
28 (0,1) : "CALL_FUNCTION_KW",
29 (1,1) : "CALL_FUNCTION_VAR_KW",
32 LOOP = 1
33 EXCEPT = 2
34 TRY_FINALLY = 3
35 END_FINALLY = 4
37 def compileFile(filename, display=0):
38 f = open(filename, 'U')
39 buf = f.read()
40 f.close()
41 mod = Module(buf, filename)
42 try:
43 mod.compile(display)
44 except SyntaxError:
45 raise
46 else:
47 f = open(filename + "c", "wb")
48 mod.dump(f)
49 f.close()
51 def compile(source, filename, mode, flags=None, dont_inherit=None):
52 """Replacement for builtin compile() function"""
53 if flags is not None or dont_inherit is not None:
54 raise RuntimeError, "not implemented yet"
56 if mode == "single":
57 gen = Interactive(source, filename)
58 elif mode == "exec":
59 gen = Module(source, filename)
60 elif mode == "eval":
61 gen = Expression(source, filename)
62 else:
63 raise ValueError("compile() 3rd arg must be 'exec' or "
64 "'eval' or 'single'")
65 gen.compile()
66 return gen.code
68 class AbstractCompileMode:
70 mode = None # defined by subclass
72 def __init__(self, source, filename):
73 self.source = source
74 self.filename = filename
75 self.code = None
77 def _get_tree(self):
78 tree = parse(self.source, self.mode)
79 misc.set_filename(self.filename, tree)
80 syntax.check(tree)
81 return tree
83 def compile(self):
84 pass # implemented by subclass
86 def getCode(self):
87 return self.code
89 class Expression(AbstractCompileMode):
91 mode = "eval"
93 def compile(self):
94 tree = self._get_tree()
95 gen = ExpressionCodeGenerator(tree)
96 self.code = gen.getCode()
98 class Interactive(AbstractCompileMode):
100 mode = "single"
102 def compile(self):
103 tree = self._get_tree()
104 gen = InteractiveCodeGenerator(tree)
105 self.code = gen.getCode()
107 class Module(AbstractCompileMode):
109 mode = "exec"
111 def compile(self, display=0):
112 tree = self._get_tree()
113 gen = ModuleCodeGenerator(tree)
114 if display:
115 import pprint
116 print pprint.pprint(tree)
117 self.code = gen.getCode()
119 def dump(self, f):
120 f.write(self.getPycHeader())
121 marshal.dump(self.code, f)
123 MAGIC = imp.get_magic()
125 def getPycHeader(self):
126 # compile.c uses marshal to write a long directly, with
127 # calling the interface that would also generate a 1-byte code
128 # to indicate the type of the value. simplest way to get the
129 # same effect is to call marshal and then skip the code.
130 mtime = os.path.getmtime(self.filename)
131 mtime = struct.pack('<i', mtime)
132 return self.MAGIC + mtime
134 class LocalNameFinder:
135 """Find local names in scope"""
136 def __init__(self, names=()):
137 self.names = misc.Set()
138 self.globals = misc.Set()
139 for name in names:
140 self.names.add(name)
142 # XXX list comprehensions and for loops
144 def getLocals(self):
145 for elt in self.globals.elements():
146 if self.names.has_elt(elt):
147 self.names.remove(elt)
148 return self.names
150 def visitDict(self, node):
151 pass
153 def visitGlobal(self, node):
154 for name in node.names:
155 self.globals.add(name)
157 def visitFunction(self, node):
158 self.names.add(node.name)
160 def visitLambda(self, node):
161 pass
163 def visitImport(self, node):
164 for name, alias in node.names:
165 self.names.add(alias or name)
167 def visitFrom(self, node):
168 for name, alias in node.names:
169 self.names.add(alias or name)
171 def visitClass(self, node):
172 self.names.add(node.name)
174 def visitAssName(self, node):
175 self.names.add(node.name)
177 def is_constant_false(node):
178 if isinstance(node, ast.Const):
179 if not node.value:
180 return 1
181 return 0
183 class CodeGenerator:
184 """Defines basic code generator for Python bytecode
186 This class is an abstract base class. Concrete subclasses must
187 define an __init__() that defines self.graph and then calls the
188 __init__() defined in this class.
190 The concrete class must also define the class attributes
191 NameFinder, FunctionGen, and ClassGen. These attributes can be
192 defined in the initClass() method, which is a hook for
193 initializing these methods after all the classes have been
194 defined.
197 optimized = 0 # is namespace access optimized?
198 __initialized = None
199 class_name = None # provide default for instance variable
201 def __init__(self):
202 if self.__initialized is None:
203 self.initClass()
204 self.__class__.__initialized = 1
205 self.checkClass()
206 self.locals = misc.Stack()
207 self.setups = misc.Stack()
208 self.last_lineno = None
209 self._setupGraphDelegation()
210 self._div_op = "BINARY_DIVIDE"
212 # XXX set flags based on future features
213 futures = self.get_module().futures
214 for feature in futures:
215 if feature == "division":
216 self.graph.setFlag(CO_FUTURE_DIVISION)
217 self._div_op = "BINARY_TRUE_DIVIDE"
218 elif feature == "absolute_import":
219 self.graph.setFlag(CO_FUTURE_ABSIMPORT)
220 elif feature == "with_statement":
221 self.graph.setFlag(CO_FUTURE_WITH_STATEMENT)
222 elif feature == "print_function":
223 self.graph.setFlag(CO_FUTURE_PRINT_FUNCTION)
225 def initClass(self):
226 """This method is called once for each class"""
228 def checkClass(self):
229 """Verify that class is constructed correctly"""
230 try:
231 assert hasattr(self, 'graph')
232 assert getattr(self, 'NameFinder')
233 assert getattr(self, 'FunctionGen')
234 assert getattr(self, 'ClassGen')
235 except AssertionError, msg:
236 intro = "Bad class construction for %s" % self.__class__.__name__
237 raise AssertionError, intro
239 def _setupGraphDelegation(self):
240 self.emit = self.graph.emit
241 self.newBlock = self.graph.newBlock
242 self.startBlock = self.graph.startBlock
243 self.nextBlock = self.graph.nextBlock
244 self.setDocstring = self.graph.setDocstring
246 def getCode(self):
247 """Return a code object"""
248 return self.graph.getCode()
250 def mangle(self, name):
251 if self.class_name is not None:
252 return misc.mangle(name, self.class_name)
253 else:
254 return name
256 def parseSymbols(self, tree):
257 s = symbols.SymbolVisitor()
258 walk(tree, s)
259 return s.scopes
261 def get_module(self):
262 raise RuntimeError, "should be implemented by subclasses"
264 # Next five methods handle name access
266 def isLocalName(self, name):
267 return self.locals.top().has_elt(name)
269 def storeName(self, name):
270 self._nameOp('STORE', name)
272 def loadName(self, name):
273 self._nameOp('LOAD', name)
275 def delName(self, name):
276 self._nameOp('DELETE', name)
278 def _nameOp(self, prefix, name):
279 name = self.mangle(name)
280 scope = self.scope.check_name(name)
281 if scope == SC_LOCAL:
282 if not self.optimized:
283 self.emit(prefix + '_NAME', name)
284 else:
285 self.emit(prefix + '_FAST', name)
286 elif scope == SC_GLOBAL_EXPLICT:
287 self.emit(prefix + '_GLOBAL', name)
288 elif scope == SC_GLOBAL_IMPLICIT:
289 if not self.optimized:
290 self.emit(prefix + '_NAME', name)
291 else:
292 self.emit(prefix + '_GLOBAL', name)
293 elif scope == SC_FREE or scope == SC_CELL:
294 self.emit(prefix + '_DEREF', name)
295 else:
296 raise RuntimeError, "unsupported scope for var %s: %d" % \
297 (name, scope)
299 def _implicitNameOp(self, prefix, name):
300 """Emit name ops for names generated implicitly by for loops
302 The interpreter generates names that start with a period or
303 dollar sign. The symbol table ignores these names because
304 they aren't present in the program text.
306 if self.optimized:
307 self.emit(prefix + '_FAST', name)
308 else:
309 self.emit(prefix + '_NAME', name)
311 # The set_lineno() function and the explicit emit() calls for
312 # SET_LINENO below are only used to generate the line number table.
313 # As of Python 2.3, the interpreter does not have a SET_LINENO
314 # instruction. pyassem treats SET_LINENO opcodes as a special case.
316 def set_lineno(self, node, force=False):
317 """Emit SET_LINENO if necessary.
319 The instruction is considered necessary if the node has a
320 lineno attribute and it is different than the last lineno
321 emitted.
323 Returns true if SET_LINENO was emitted.
325 There are no rules for when an AST node should have a lineno
326 attribute. The transformer and AST code need to be reviewed
327 and a consistent policy implemented and documented. Until
328 then, this method works around missing line numbers.
330 lineno = getattr(node, 'lineno', None)
331 if lineno is not None and (lineno != self.last_lineno
332 or force):
333 self.emit('SET_LINENO', lineno)
334 self.last_lineno = lineno
335 return True
336 return False
338 # The first few visitor methods handle nodes that generator new
339 # code objects. They use class attributes to determine what
340 # specialized code generators to use.
342 NameFinder = LocalNameFinder
343 FunctionGen = None
344 ClassGen = None
346 def visitModule(self, node):
347 self.scopes = self.parseSymbols(node)
348 self.scope = self.scopes[node]
349 self.emit('SET_LINENO', 0)
350 if node.doc:
351 self.emit('LOAD_CONST', node.doc)
352 self.storeName('__doc__')
353 lnf = walk(node.node, self.NameFinder(), verbose=0)
354 self.locals.push(lnf.getLocals())
355 self.visit(node.node)
356 self.emit('LOAD_CONST', None)
357 self.emit('RETURN_VALUE')
359 def visitExpression(self, node):
360 self.set_lineno(node)
361 self.scopes = self.parseSymbols(node)
362 self.scope = self.scopes[node]
363 self.visit(node.node)
364 self.emit('RETURN_VALUE')
366 def visitFunction(self, node):
367 self._visitFuncOrLambda(node, isLambda=0)
368 if node.doc:
369 self.setDocstring(node.doc)
370 self.storeName(node.name)
372 def visitLambda(self, node):
373 self._visitFuncOrLambda(node, isLambda=1)
375 def _visitFuncOrLambda(self, node, isLambda=0):
376 if not isLambda and node.decorators:
377 for decorator in node.decorators.nodes:
378 self.visit(decorator)
379 ndecorators = len(node.decorators.nodes)
380 else:
381 ndecorators = 0
383 gen = self.FunctionGen(node, self.scopes, isLambda,
384 self.class_name, self.get_module())
385 walk(node.code, gen)
386 gen.finish()
387 self.set_lineno(node)
388 for default in node.defaults:
389 self.visit(default)
390 self._makeClosure(gen, len(node.defaults))
391 for i in range(ndecorators):
392 self.emit('CALL_FUNCTION', 1)
394 def visitClass(self, node):
395 gen = self.ClassGen(node, self.scopes,
396 self.get_module())
397 walk(node.code, gen)
398 gen.finish()
399 self.set_lineno(node)
400 self.emit('LOAD_CONST', node.name)
401 for base in node.bases:
402 self.visit(base)
403 self.emit('BUILD_TUPLE', len(node.bases))
404 self._makeClosure(gen, 0)
405 self.emit('CALL_FUNCTION', 0)
406 self.emit('BUILD_CLASS')
407 self.storeName(node.name)
409 # The rest are standard visitor methods
411 # The next few implement control-flow statements
413 def visitIf(self, node):
414 end = self.newBlock()
415 numtests = len(node.tests)
416 for i in range(numtests):
417 test, suite = node.tests[i]
418 if is_constant_false(test):
419 # XXX will need to check generator stuff here
420 continue
421 self.set_lineno(test)
422 self.visit(test)
423 nextTest = self.newBlock()
424 self.emit('POP_JUMP_IF_FALSE', nextTest)
425 self.nextBlock()
426 self.visit(suite)
427 self.emit('JUMP_FORWARD', end)
428 self.startBlock(nextTest)
429 if node.else_:
430 self.visit(node.else_)
431 self.nextBlock(end)
433 def visitWhile(self, node):
434 self.set_lineno(node)
436 loop = self.newBlock()
437 else_ = self.newBlock()
439 after = self.newBlock()
440 self.emit('SETUP_LOOP', after)
442 self.nextBlock(loop)
443 self.setups.push((LOOP, loop))
445 self.set_lineno(node, force=True)
446 self.visit(node.test)
447 self.emit('POP_JUMP_IF_FALSE', else_ or after)
449 self.nextBlock()
450 self.visit(node.body)
451 self.emit('JUMP_ABSOLUTE', loop)
453 self.startBlock(else_) # or just the POPs if not else clause
454 self.emit('POP_BLOCK')
455 self.setups.pop()
456 if node.else_:
457 self.visit(node.else_)
458 self.nextBlock(after)
460 def visitFor(self, node):
461 start = self.newBlock()
462 anchor = self.newBlock()
463 after = self.newBlock()
464 self.setups.push((LOOP, start))
466 self.set_lineno(node)
467 self.emit('SETUP_LOOP', after)
468 self.visit(node.list)
469 self.emit('GET_ITER')
471 self.nextBlock(start)
472 self.set_lineno(node, force=1)
473 self.emit('FOR_ITER', anchor)
474 self.visit(node.assign)
475 self.visit(node.body)
476 self.emit('JUMP_ABSOLUTE', start)
477 self.nextBlock(anchor)
478 self.emit('POP_BLOCK')
479 self.setups.pop()
480 if node.else_:
481 self.visit(node.else_)
482 self.nextBlock(after)
484 def visitBreak(self, node):
485 if not self.setups:
486 raise SyntaxError, "'break' outside loop (%s, %d)" % \
487 (node.filename, node.lineno)
488 self.set_lineno(node)
489 self.emit('BREAK_LOOP')
491 def visitContinue(self, node):
492 if not self.setups:
493 raise SyntaxError, "'continue' outside loop (%s, %d)" % \
494 (node.filename, node.lineno)
495 kind, block = self.setups.top()
496 if kind == LOOP:
497 self.set_lineno(node)
498 self.emit('JUMP_ABSOLUTE', block)
499 self.nextBlock()
500 elif kind == EXCEPT or kind == TRY_FINALLY:
501 self.set_lineno(node)
502 # find the block that starts the loop
503 top = len(self.setups)
504 while top > 0:
505 top = top - 1
506 kind, loop_block = self.setups[top]
507 if kind == LOOP:
508 break
509 if kind != LOOP:
510 raise SyntaxError, "'continue' outside loop (%s, %d)" % \
511 (node.filename, node.lineno)
512 self.emit('CONTINUE_LOOP', loop_block)
513 self.nextBlock()
514 elif kind == END_FINALLY:
515 msg = "'continue' not allowed inside 'finally' clause (%s, %d)"
516 raise SyntaxError, msg % (node.filename, node.lineno)
518 def visitTest(self, node, jump):
519 end = self.newBlock()
520 for child in node.nodes[:-1]:
521 self.visit(child)
522 self.emit(jump, end)
523 self.nextBlock()
524 self.visit(node.nodes[-1])
525 self.nextBlock(end)
527 def visitAnd(self, node):
528 self.visitTest(node, 'JUMP_IF_FALSE_OR_POP')
530 def visitOr(self, node):
531 self.visitTest(node, 'JUMP_IF_TRUE_OR_POP')
533 def visitIfExp(self, node):
534 endblock = self.newBlock()
535 elseblock = self.newBlock()
536 self.visit(node.test)
537 self.emit('POP_JUMP_IF_FALSE', elseblock)
538 self.visit(node.then)
539 self.emit('JUMP_FORWARD', endblock)
540 self.nextBlock(elseblock)
541 self.visit(node.else_)
542 self.nextBlock(endblock)
544 def visitCompare(self, node):
545 self.visit(node.expr)
546 cleanup = self.newBlock()
547 for op, code in node.ops[:-1]:
548 self.visit(code)
549 self.emit('DUP_TOP')
550 self.emit('ROT_THREE')
551 self.emit('COMPARE_OP', op)
552 self.emit('JUMP_IF_FALSE_OR_POP', cleanup)
553 self.nextBlock()
554 # now do the last comparison
555 if node.ops:
556 op, code = node.ops[-1]
557 self.visit(code)
558 self.emit('COMPARE_OP', op)
559 if len(node.ops) > 1:
560 end = self.newBlock()
561 self.emit('JUMP_FORWARD', end)
562 self.startBlock(cleanup)
563 self.emit('ROT_TWO')
564 self.emit('POP_TOP')
565 self.nextBlock(end)
567 # list comprehensions
568 def visitListComp(self, node):
569 self.set_lineno(node)
570 # setup list
571 self.emit('BUILD_LIST', 0)
573 stack = []
574 for i, for_ in zip(range(len(node.quals)), node.quals):
575 start, anchor = self.visit(for_)
576 cont = None
577 for if_ in for_.ifs:
578 if cont is None:
579 cont = self.newBlock()
580 self.visit(if_, cont)
581 stack.insert(0, (start, cont, anchor))
583 self.visit(node.expr)
584 self.emit('LIST_APPEND', len(node.quals) + 1)
586 for start, cont, anchor in stack:
587 if cont:
588 self.nextBlock(cont)
589 self.emit('JUMP_ABSOLUTE', start)
590 self.startBlock(anchor)
592 def visitListCompFor(self, node):
593 start = self.newBlock()
594 anchor = self.newBlock()
596 self.visit(node.list)
597 self.emit('GET_ITER')
598 self.nextBlock(start)
599 self.set_lineno(node, force=True)
600 self.emit('FOR_ITER', anchor)
601 self.nextBlock()
602 self.visit(node.assign)
603 return start, anchor
605 def visitListCompIf(self, node, branch):
606 self.set_lineno(node, force=True)
607 self.visit(node.test)
608 self.emit('POP_JUMP_IF_FALSE', branch)
609 self.newBlock()
611 def _makeClosure(self, gen, args):
612 frees = gen.scope.get_free_vars()
613 if frees:
614 for name in frees:
615 self.emit('LOAD_CLOSURE', name)
616 self.emit('BUILD_TUPLE', len(frees))
617 self.emit('LOAD_CONST', gen)
618 self.emit('MAKE_CLOSURE', args)
619 else:
620 self.emit('LOAD_CONST', gen)
621 self.emit('MAKE_FUNCTION', args)
623 def visitGenExpr(self, node):
624 gen = GenExprCodeGenerator(node, self.scopes, self.class_name,
625 self.get_module())
626 walk(node.code, gen)
627 gen.finish()
628 self.set_lineno(node)
629 self._makeClosure(gen, 0)
630 # precomputation of outmost iterable
631 self.visit(node.code.quals[0].iter)
632 self.emit('GET_ITER')
633 self.emit('CALL_FUNCTION', 1)
635 def visitGenExprInner(self, node):
636 self.set_lineno(node)
637 # setup list
639 stack = []
640 for i, for_ in zip(range(len(node.quals)), node.quals):
641 start, anchor, end = self.visit(for_)
642 cont = None
643 for if_ in for_.ifs:
644 if cont is None:
645 cont = self.newBlock()
646 self.visit(if_, cont)
647 stack.insert(0, (start, cont, anchor, end))
649 self.visit(node.expr)
650 self.emit('YIELD_VALUE')
651 self.emit('POP_TOP')
653 for start, cont, anchor, end in stack:
654 if cont:
655 self.nextBlock(cont)
656 self.emit('JUMP_ABSOLUTE', start)
657 self.startBlock(anchor)
658 self.emit('POP_BLOCK')
659 self.setups.pop()
660 self.nextBlock(end)
662 self.emit('LOAD_CONST', None)
664 def visitGenExprFor(self, node):
665 start = self.newBlock()
666 anchor = self.newBlock()
667 end = self.newBlock()
669 self.setups.push((LOOP, start))
670 self.emit('SETUP_LOOP', end)
672 if node.is_outmost:
673 self.loadName('.0')
674 else:
675 self.visit(node.iter)
676 self.emit('GET_ITER')
678 self.nextBlock(start)
679 self.set_lineno(node, force=True)
680 self.emit('FOR_ITER', anchor)
681 self.nextBlock()
682 self.visit(node.assign)
683 return start, anchor, end
685 def visitGenExprIf(self, node, branch):
686 self.set_lineno(node, force=True)
687 self.visit(node.test)
688 self.emit('POP_JUMP_IF_FALSE', branch)
689 self.newBlock()
691 # exception related
693 def visitAssert(self, node):
694 # XXX would be interesting to implement this via a
695 # transformation of the AST before this stage
696 if __debug__:
697 end = self.newBlock()
698 self.set_lineno(node)
699 # XXX AssertionError appears to be special case -- it is always
700 # loaded as a global even if there is a local name. I guess this
701 # is a sort of renaming op.
702 self.nextBlock()
703 self.visit(node.test)
704 self.emit('POP_JUMP_IF_TRUE', end)
705 self.nextBlock()
706 self.emit('LOAD_GLOBAL', 'AssertionError')
707 if node.fail:
708 self.visit(node.fail)
709 self.emit('RAISE_VARARGS', 2)
710 else:
711 self.emit('RAISE_VARARGS', 1)
712 self.nextBlock(end)
714 def visitRaise(self, node):
715 self.set_lineno(node)
716 n = 0
717 if node.expr1:
718 self.visit(node.expr1)
719 n = n + 1
720 if node.expr2:
721 self.visit(node.expr2)
722 n = n + 1
723 if node.expr3:
724 self.visit(node.expr3)
725 n = n + 1
726 self.emit('RAISE_VARARGS', n)
728 def visitTryExcept(self, node):
729 body = self.newBlock()
730 handlers = self.newBlock()
731 end = self.newBlock()
732 if node.else_:
733 lElse = self.newBlock()
734 else:
735 lElse = end
736 self.set_lineno(node)
737 self.emit('SETUP_EXCEPT', handlers)
738 self.nextBlock(body)
739 self.setups.push((EXCEPT, body))
740 self.visit(node.body)
741 self.emit('POP_BLOCK')
742 self.setups.pop()
743 self.emit('JUMP_FORWARD', lElse)
744 self.startBlock(handlers)
746 last = len(node.handlers) - 1
747 for i in range(len(node.handlers)):
748 expr, target, body = node.handlers[i]
749 self.set_lineno(expr)
750 if expr:
751 self.emit('DUP_TOP')
752 self.visit(expr)
753 self.emit('COMPARE_OP', 'exception match')
754 next = self.newBlock()
755 self.emit('POP_JUMP_IF_FALSE', next)
756 self.nextBlock()
757 self.emit('POP_TOP')
758 if target:
759 self.visit(target)
760 else:
761 self.emit('POP_TOP')
762 self.emit('POP_TOP')
763 self.visit(body)
764 self.emit('JUMP_FORWARD', end)
765 if expr:
766 self.nextBlock(next)
767 else:
768 self.nextBlock()
769 self.emit('END_FINALLY')
770 if node.else_:
771 self.nextBlock(lElse)
772 self.visit(node.else_)
773 self.nextBlock(end)
775 def visitTryFinally(self, node):
776 body = self.newBlock()
777 final = self.newBlock()
778 self.set_lineno(node)
779 self.emit('SETUP_FINALLY', final)
780 self.nextBlock(body)
781 self.setups.push((TRY_FINALLY, body))
782 self.visit(node.body)
783 self.emit('POP_BLOCK')
784 self.setups.pop()
785 self.emit('LOAD_CONST', None)
786 self.nextBlock(final)
787 self.setups.push((END_FINALLY, final))
788 self.visit(node.final)
789 self.emit('END_FINALLY')
790 self.setups.pop()
792 __with_count = 0
794 def visitWith(self, node):
795 body = self.newBlock()
796 final = self.newBlock()
797 self.__with_count += 1
798 valuevar = "_[%d]" % self.__with_count
799 self.set_lineno(node)
800 self.visit(node.expr)
801 self.emit('DUP_TOP')
802 self.emit('LOAD_ATTR', '__exit__')
803 self.emit('ROT_TWO')
804 self.emit('LOAD_ATTR', '__enter__')
805 self.emit('CALL_FUNCTION', 0)
806 if node.vars is None:
807 self.emit('POP_TOP')
808 else:
809 self._implicitNameOp('STORE', valuevar)
810 self.emit('SETUP_FINALLY', final)
811 self.nextBlock(body)
812 self.setups.push((TRY_FINALLY, body))
813 if node.vars is not None:
814 self._implicitNameOp('LOAD', valuevar)
815 self._implicitNameOp('DELETE', valuevar)
816 self.visit(node.vars)
817 self.visit(node.body)
818 self.emit('POP_BLOCK')
819 self.setups.pop()
820 self.emit('LOAD_CONST', None)
821 self.nextBlock(final)
822 self.setups.push((END_FINALLY, final))
823 self.emit('WITH_CLEANUP')
824 self.emit('END_FINALLY')
825 self.setups.pop()
826 self.__with_count -= 1
828 # misc
830 def visitDiscard(self, node):
831 self.set_lineno(node)
832 self.visit(node.expr)
833 self.emit('POP_TOP')
835 def visitConst(self, node):
836 self.emit('LOAD_CONST', node.value)
838 def visitKeyword(self, node):
839 self.emit('LOAD_CONST', node.name)
840 self.visit(node.expr)
842 def visitGlobal(self, node):
843 # no code to generate
844 pass
846 def visitName(self, node):
847 self.set_lineno(node)
848 self.loadName(node.name)
850 def visitPass(self, node):
851 self.set_lineno(node)
853 def visitImport(self, node):
854 self.set_lineno(node)
855 level = 0 if self.graph.checkFlag(CO_FUTURE_ABSIMPORT) else -1
856 for name, alias in node.names:
857 if VERSION > 1:
858 self.emit('LOAD_CONST', level)
859 self.emit('LOAD_CONST', None)
860 self.emit('IMPORT_NAME', name)
861 mod = name.split(".")[0]
862 if alias:
863 self._resolveDots(name)
864 self.storeName(alias)
865 else:
866 self.storeName(mod)
868 def visitFrom(self, node):
869 self.set_lineno(node)
870 level = node.level
871 if level == 0 and not self.graph.checkFlag(CO_FUTURE_ABSIMPORT):
872 level = -1
873 fromlist = map(lambda (name, alias): name, node.names)
874 if VERSION > 1:
875 self.emit('LOAD_CONST', level)
876 self.emit('LOAD_CONST', tuple(fromlist))
877 self.emit('IMPORT_NAME', node.modname)
878 for name, alias in node.names:
879 if VERSION > 1:
880 if name == '*':
881 self.namespace = 0
882 self.emit('IMPORT_STAR')
883 # There can only be one name w/ from ... import *
884 assert len(node.names) == 1
885 return
886 else:
887 self.emit('IMPORT_FROM', name)
888 self._resolveDots(name)
889 self.storeName(alias or name)
890 else:
891 self.emit('IMPORT_FROM', name)
892 self.emit('POP_TOP')
894 def _resolveDots(self, name):
895 elts = name.split(".")
896 if len(elts) == 1:
897 return
898 for elt in elts[1:]:
899 self.emit('LOAD_ATTR', elt)
901 def visitGetattr(self, node):
902 self.visit(node.expr)
903 self.emit('LOAD_ATTR', self.mangle(node.attrname))
905 # next five implement assignments
907 def visitAssign(self, node):
908 self.set_lineno(node)
909 self.visit(node.expr)
910 dups = len(node.nodes) - 1
911 for i in range(len(node.nodes)):
912 elt = node.nodes[i]
913 if i < dups:
914 self.emit('DUP_TOP')
915 if isinstance(elt, ast.Node):
916 self.visit(elt)
918 def visitAssName(self, node):
919 if node.flags == 'OP_ASSIGN':
920 self.storeName(node.name)
921 elif node.flags == 'OP_DELETE':
922 self.set_lineno(node)
923 self.delName(node.name)
924 else:
925 print "oops", node.flags
927 def visitAssAttr(self, node):
928 self.visit(node.expr)
929 if node.flags == 'OP_ASSIGN':
930 self.emit('STORE_ATTR', self.mangle(node.attrname))
931 elif node.flags == 'OP_DELETE':
932 self.emit('DELETE_ATTR', self.mangle(node.attrname))
933 else:
934 print "warning: unexpected flags:", node.flags
935 print node
937 def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
938 if findOp(node) != 'OP_DELETE':
939 self.emit(op, len(node.nodes))
940 for child in node.nodes:
941 self.visit(child)
943 if VERSION > 1:
944 visitAssTuple = _visitAssSequence
945 visitAssList = _visitAssSequence
946 else:
947 def visitAssTuple(self, node):
948 self._visitAssSequence(node, 'UNPACK_TUPLE')
950 def visitAssList(self, node):
951 self._visitAssSequence(node, 'UNPACK_LIST')
953 # augmented assignment
955 def visitAugAssign(self, node):
956 self.set_lineno(node)
957 aug_node = wrap_aug(node.node)
958 self.visit(aug_node, "load")
959 self.visit(node.expr)
960 self.emit(self._augmented_opcode[node.op])
961 self.visit(aug_node, "store")
963 _augmented_opcode = {
964 '+=' : 'INPLACE_ADD',
965 '-=' : 'INPLACE_SUBTRACT',
966 '*=' : 'INPLACE_MULTIPLY',
967 '/=' : 'INPLACE_DIVIDE',
968 '//=': 'INPLACE_FLOOR_DIVIDE',
969 '%=' : 'INPLACE_MODULO',
970 '**=': 'INPLACE_POWER',
971 '>>=': 'INPLACE_RSHIFT',
972 '<<=': 'INPLACE_LSHIFT',
973 '&=' : 'INPLACE_AND',
974 '^=' : 'INPLACE_XOR',
975 '|=' : 'INPLACE_OR',
978 def visitAugName(self, node, mode):
979 if mode == "load":
980 self.loadName(node.name)
981 elif mode == "store":
982 self.storeName(node.name)
984 def visitAugGetattr(self, node, mode):
985 if mode == "load":
986 self.visit(node.expr)
987 self.emit('DUP_TOP')
988 self.emit('LOAD_ATTR', self.mangle(node.attrname))
989 elif mode == "store":
990 self.emit('ROT_TWO')
991 self.emit('STORE_ATTR', self.mangle(node.attrname))
993 def visitAugSlice(self, node, mode):
994 if mode == "load":
995 self.visitSlice(node, 1)
996 elif mode == "store":
997 slice = 0
998 if node.lower:
999 slice = slice | 1
1000 if node.upper:
1001 slice = slice | 2
1002 if slice == 0:
1003 self.emit('ROT_TWO')
1004 elif slice == 3:
1005 self.emit('ROT_FOUR')
1006 else:
1007 self.emit('ROT_THREE')
1008 self.emit('STORE_SLICE+%d' % slice)
1010 def visitAugSubscript(self, node, mode):
1011 if mode == "load":
1012 self.visitSubscript(node, 1)
1013 elif mode == "store":
1014 self.emit('ROT_THREE')
1015 self.emit('STORE_SUBSCR')
1017 def visitExec(self, node):
1018 self.visit(node.expr)
1019 if node.locals is None:
1020 self.emit('LOAD_CONST', None)
1021 else:
1022 self.visit(node.locals)
1023 if node.globals is None:
1024 self.emit('DUP_TOP')
1025 else:
1026 self.visit(node.globals)
1027 self.emit('EXEC_STMT')
1029 def visitCallFunc(self, node):
1030 pos = 0
1031 kw = 0
1032 self.set_lineno(node)
1033 self.visit(node.node)
1034 for arg in node.args:
1035 self.visit(arg)
1036 if isinstance(arg, ast.Keyword):
1037 kw = kw + 1
1038 else:
1039 pos = pos + 1
1040 if node.star_args is not None:
1041 self.visit(node.star_args)
1042 if node.dstar_args is not None:
1043 self.visit(node.dstar_args)
1044 have_star = node.star_args is not None
1045 have_dstar = node.dstar_args is not None
1046 opcode = callfunc_opcode_info[have_star, have_dstar]
1047 self.emit(opcode, kw << 8 | pos)
1049 def visitPrint(self, node, newline=0):
1050 self.set_lineno(node)
1051 if node.dest:
1052 self.visit(node.dest)
1053 for child in node.nodes:
1054 if node.dest:
1055 self.emit('DUP_TOP')
1056 self.visit(child)
1057 if node.dest:
1058 self.emit('ROT_TWO')
1059 self.emit('PRINT_ITEM_TO')
1060 else:
1061 self.emit('PRINT_ITEM')
1062 if node.dest and not newline:
1063 self.emit('POP_TOP')
1065 def visitPrintnl(self, node):
1066 self.visitPrint(node, newline=1)
1067 if node.dest:
1068 self.emit('PRINT_NEWLINE_TO')
1069 else:
1070 self.emit('PRINT_NEWLINE')
1072 def visitReturn(self, node):
1073 self.set_lineno(node)
1074 self.visit(node.value)
1075 self.emit('RETURN_VALUE')
1077 def visitYield(self, node):
1078 self.set_lineno(node)
1079 self.visit(node.value)
1080 self.emit('YIELD_VALUE')
1082 # slice and subscript stuff
1084 def visitSlice(self, node, aug_flag=None):
1085 # aug_flag is used by visitAugSlice
1086 self.visit(node.expr)
1087 slice = 0
1088 if node.lower:
1089 self.visit(node.lower)
1090 slice = slice | 1
1091 if node.upper:
1092 self.visit(node.upper)
1093 slice = slice | 2
1094 if aug_flag:
1095 if slice == 0:
1096 self.emit('DUP_TOP')
1097 elif slice == 3:
1098 self.emit('DUP_TOPX', 3)
1099 else:
1100 self.emit('DUP_TOPX', 2)
1101 if node.flags == 'OP_APPLY':
1102 self.emit('SLICE+%d' % slice)
1103 elif node.flags == 'OP_ASSIGN':
1104 self.emit('STORE_SLICE+%d' % slice)
1105 elif node.flags == 'OP_DELETE':
1106 self.emit('DELETE_SLICE+%d' % slice)
1107 else:
1108 print "weird slice", node.flags
1109 raise
1111 def visitSubscript(self, node, aug_flag=None):
1112 self.visit(node.expr)
1113 for sub in node.subs:
1114 self.visit(sub)
1115 if len(node.subs) > 1:
1116 self.emit('BUILD_TUPLE', len(node.subs))
1117 if aug_flag:
1118 self.emit('DUP_TOPX', 2)
1119 if node.flags == 'OP_APPLY':
1120 self.emit('BINARY_SUBSCR')
1121 elif node.flags == 'OP_ASSIGN':
1122 self.emit('STORE_SUBSCR')
1123 elif node.flags == 'OP_DELETE':
1124 self.emit('DELETE_SUBSCR')
1126 # binary ops
1128 def binaryOp(self, node, op):
1129 self.visit(node.left)
1130 self.visit(node.right)
1131 self.emit(op)
1133 def visitAdd(self, node):
1134 return self.binaryOp(node, 'BINARY_ADD')
1136 def visitSub(self, node):
1137 return self.binaryOp(node, 'BINARY_SUBTRACT')
1139 def visitMul(self, node):
1140 return self.binaryOp(node, 'BINARY_MULTIPLY')
1142 def visitDiv(self, node):
1143 return self.binaryOp(node, self._div_op)
1145 def visitFloorDiv(self, node):
1146 return self.binaryOp(node, 'BINARY_FLOOR_DIVIDE')
1148 def visitMod(self, node):
1149 return self.binaryOp(node, 'BINARY_MODULO')
1151 def visitPower(self, node):
1152 return self.binaryOp(node, 'BINARY_POWER')
1154 def visitLeftShift(self, node):
1155 return self.binaryOp(node, 'BINARY_LSHIFT')
1157 def visitRightShift(self, node):
1158 return self.binaryOp(node, 'BINARY_RSHIFT')
1160 # unary ops
1162 def unaryOp(self, node, op):
1163 self.visit(node.expr)
1164 self.emit(op)
1166 def visitInvert(self, node):
1167 return self.unaryOp(node, 'UNARY_INVERT')
1169 def visitUnarySub(self, node):
1170 return self.unaryOp(node, 'UNARY_NEGATIVE')
1172 def visitUnaryAdd(self, node):
1173 return self.unaryOp(node, 'UNARY_POSITIVE')
1175 def visitUnaryInvert(self, node):
1176 return self.unaryOp(node, 'UNARY_INVERT')
1178 def visitNot(self, node):
1179 return self.unaryOp(node, 'UNARY_NOT')
1181 def visitBackquote(self, node):
1182 return self.unaryOp(node, 'UNARY_CONVERT')
1184 # bit ops
1186 def bitOp(self, nodes, op):
1187 self.visit(nodes[0])
1188 for node in nodes[1:]:
1189 self.visit(node)
1190 self.emit(op)
1192 def visitBitand(self, node):
1193 return self.bitOp(node.nodes, 'BINARY_AND')
1195 def visitBitor(self, node):
1196 return self.bitOp(node.nodes, 'BINARY_OR')
1198 def visitBitxor(self, node):
1199 return self.bitOp(node.nodes, 'BINARY_XOR')
1201 # object constructors
1203 def visitEllipsis(self, node):
1204 self.emit('LOAD_CONST', Ellipsis)
1206 def visitTuple(self, node):
1207 self.set_lineno(node)
1208 for elt in node.nodes:
1209 self.visit(elt)
1210 self.emit('BUILD_TUPLE', len(node.nodes))
1212 def visitList(self, node):
1213 self.set_lineno(node)
1214 for elt in node.nodes:
1215 self.visit(elt)
1216 self.emit('BUILD_LIST', len(node.nodes))
1218 def visitSliceobj(self, node):
1219 for child in node.nodes:
1220 self.visit(child)
1221 self.emit('BUILD_SLICE', len(node.nodes))
1223 def visitDict(self, node):
1224 self.set_lineno(node)
1225 self.emit('BUILD_MAP', 0)
1226 for k, v in node.items:
1227 self.emit('DUP_TOP')
1228 self.visit(k)
1229 self.visit(v)
1230 self.emit('ROT_THREE')
1231 self.emit('STORE_SUBSCR')
1233 class NestedScopeMixin:
1234 """Defines initClass() for nested scoping (Python 2.2-compatible)"""
1235 def initClass(self):
1236 self.__class__.NameFinder = LocalNameFinder
1237 self.__class__.FunctionGen = FunctionCodeGenerator
1238 self.__class__.ClassGen = ClassCodeGenerator
1240 class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator):
1241 __super_init = CodeGenerator.__init__
1243 scopes = None
1245 def __init__(self, tree):
1246 self.graph = pyassem.PyFlowGraph("<module>", tree.filename)
1247 self.futures = future.find_futures(tree)
1248 self.__super_init()
1249 walk(tree, self)
1251 def get_module(self):
1252 return self
1254 class ExpressionCodeGenerator(NestedScopeMixin, CodeGenerator):
1255 __super_init = CodeGenerator.__init__
1257 scopes = None
1258 futures = ()
1260 def __init__(self, tree):
1261 self.graph = pyassem.PyFlowGraph("<expression>", tree.filename)
1262 self.__super_init()
1263 walk(tree, self)
1265 def get_module(self):
1266 return self
1268 class InteractiveCodeGenerator(NestedScopeMixin, CodeGenerator):
1270 __super_init = CodeGenerator.__init__
1272 scopes = None
1273 futures = ()
1275 def __init__(self, tree):
1276 self.graph = pyassem.PyFlowGraph("<interactive>", tree.filename)
1277 self.__super_init()
1278 self.set_lineno(tree)
1279 walk(tree, self)
1280 self.emit('RETURN_VALUE')
1282 def get_module(self):
1283 return self
1285 def visitDiscard(self, node):
1286 # XXX Discard means it's an expression. Perhaps this is a bad
1287 # name.
1288 self.visit(node.expr)
1289 self.emit('PRINT_EXPR')
1291 class AbstractFunctionCode:
1292 optimized = 1
1293 lambdaCount = 0
1295 def __init__(self, func, scopes, isLambda, class_name, mod):
1296 self.class_name = class_name
1297 self.module = mod
1298 if isLambda:
1299 klass = FunctionCodeGenerator
1300 name = "<lambda.%d>" % klass.lambdaCount
1301 klass.lambdaCount = klass.lambdaCount + 1
1302 else:
1303 name = func.name
1305 args, hasTupleArg = generateArgList(func.argnames)
1306 self.graph = pyassem.PyFlowGraph(name, func.filename, args,
1307 optimized=1)
1308 self.isLambda = isLambda
1309 self.super_init()
1311 if not isLambda and func.doc:
1312 self.setDocstring(func.doc)
1314 lnf = walk(func.code, self.NameFinder(args), verbose=0)
1315 self.locals.push(lnf.getLocals())
1316 if func.varargs:
1317 self.graph.setFlag(CO_VARARGS)
1318 if func.kwargs:
1319 self.graph.setFlag(CO_VARKEYWORDS)
1320 self.set_lineno(func)
1321 if hasTupleArg:
1322 self.generateArgUnpack(func.argnames)
1324 def get_module(self):
1325 return self.module
1327 def finish(self):
1328 self.graph.startExitBlock()
1329 if not self.isLambda:
1330 self.emit('LOAD_CONST', None)
1331 self.emit('RETURN_VALUE')
1333 def generateArgUnpack(self, args):
1334 for i in range(len(args)):
1335 arg = args[i]
1336 if isinstance(arg, tuple):
1337 self.emit('LOAD_FAST', '.%d' % (i * 2))
1338 self.unpackSequence(arg)
1340 def unpackSequence(self, tup):
1341 if VERSION > 1:
1342 self.emit('UNPACK_SEQUENCE', len(tup))
1343 else:
1344 self.emit('UNPACK_TUPLE', len(tup))
1345 for elt in tup:
1346 if isinstance(elt, tuple):
1347 self.unpackSequence(elt)
1348 else:
1349 self._nameOp('STORE', elt)
1351 unpackTuple = unpackSequence
1353 class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
1354 CodeGenerator):
1355 super_init = CodeGenerator.__init__ # call be other init
1356 scopes = None
1358 __super_init = AbstractFunctionCode.__init__
1360 def __init__(self, func, scopes, isLambda, class_name, mod):
1361 self.scopes = scopes
1362 self.scope = scopes[func]
1363 self.__super_init(func, scopes, isLambda, class_name, mod)
1364 self.graph.setFreeVars(self.scope.get_free_vars())
1365 self.graph.setCellVars(self.scope.get_cell_vars())
1366 if self.scope.generator is not None:
1367 self.graph.setFlag(CO_GENERATOR)
1369 class GenExprCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
1370 CodeGenerator):
1371 super_init = CodeGenerator.__init__ # call be other init
1372 scopes = None
1374 __super_init = AbstractFunctionCode.__init__
1376 def __init__(self, gexp, scopes, class_name, mod):
1377 self.scopes = scopes
1378 self.scope = scopes[gexp]
1379 self.__super_init(gexp, scopes, 1, class_name, mod)
1380 self.graph.setFreeVars(self.scope.get_free_vars())
1381 self.graph.setCellVars(self.scope.get_cell_vars())
1382 self.graph.setFlag(CO_GENERATOR)
1384 class AbstractClassCode:
1386 def __init__(self, klass, scopes, module):
1387 self.class_name = klass.name
1388 self.module = module
1389 self.graph = pyassem.PyFlowGraph(klass.name, klass.filename,
1390 optimized=0, klass=1)
1391 self.super_init()
1392 lnf = walk(klass.code, self.NameFinder(), verbose=0)
1393 self.locals.push(lnf.getLocals())
1394 self.graph.setFlag(CO_NEWLOCALS)
1395 if klass.doc:
1396 self.setDocstring(klass.doc)
1398 def get_module(self):
1399 return self.module
1401 def finish(self):
1402 self.graph.startExitBlock()
1403 self.emit('LOAD_LOCALS')
1404 self.emit('RETURN_VALUE')
1406 class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator):
1407 super_init = CodeGenerator.__init__
1408 scopes = None
1410 __super_init = AbstractClassCode.__init__
1412 def __init__(self, klass, scopes, module):
1413 self.scopes = scopes
1414 self.scope = scopes[klass]
1415 self.__super_init(klass, scopes, module)
1416 self.graph.setFreeVars(self.scope.get_free_vars())
1417 self.graph.setCellVars(self.scope.get_cell_vars())
1418 self.set_lineno(klass)
1419 self.emit("LOAD_GLOBAL", "__name__")
1420 self.storeName("__module__")
1421 if klass.doc:
1422 self.emit("LOAD_CONST", klass.doc)
1423 self.storeName('__doc__')
1425 def generateArgList(arglist):
1426 """Generate an arg list marking TupleArgs"""
1427 args = []
1428 extra = []
1429 count = 0
1430 for i in range(len(arglist)):
1431 elt = arglist[i]
1432 if isinstance(elt, str):
1433 args.append(elt)
1434 elif isinstance(elt, tuple):
1435 args.append(TupleArg(i * 2, elt))
1436 extra.extend(misc.flatten(elt))
1437 count = count + 1
1438 else:
1439 raise ValueError, "unexpect argument type:", elt
1440 return args + extra, count
1442 def findOp(node):
1443 """Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
1444 v = OpFinder()
1445 walk(node, v, verbose=0)
1446 return v.op
1448 class OpFinder:
1449 def __init__(self):
1450 self.op = None
1451 def visitAssName(self, node):
1452 if self.op is None:
1453 self.op = node.flags
1454 elif self.op != node.flags:
1455 raise ValueError, "mixed ops in stmt"
1456 visitAssAttr = visitAssName
1457 visitSubscript = visitAssName
1459 class Delegator:
1460 """Base class to support delegation for augmented assignment nodes
1462 To generator code for augmented assignments, we use the following
1463 wrapper classes. In visitAugAssign, the left-hand expression node
1464 is visited twice. The first time the visit uses the normal method
1465 for that node . The second time the visit uses a different method
1466 that generates the appropriate code to perform the assignment.
1467 These delegator classes wrap the original AST nodes in order to
1468 support the variant visit methods.
1470 def __init__(self, obj):
1471 self.obj = obj
1473 def __getattr__(self, attr):
1474 return getattr(self.obj, attr)
1476 class AugGetattr(Delegator):
1477 pass
1479 class AugName(Delegator):
1480 pass
1482 class AugSlice(Delegator):
1483 pass
1485 class AugSubscript(Delegator):
1486 pass
1488 wrapper = {
1489 ast.Getattr: AugGetattr,
1490 ast.Name: AugName,
1491 ast.Slice: AugSlice,
1492 ast.Subscript: AugSubscript,
1495 def wrap_aug(node):
1496 return wrapper[node.__class__](node)
1498 if __name__ == "__main__":
1499 for file in sys.argv[1:]:
1500 compileFile(file)