1 """Module symbol-table generator"""
3 from compiler
import ast
4 from compiler
.consts
import SC_LOCAL
, SC_GLOBAL_IMPLICIT
, SC_GLOBAL_EXPLICT
, \
5 SC_FREE
, SC_CELL
, SC_UNKNOWN
6 from compiler
.misc
import mangle
15 # XXX how much information do I need about each name?
16 def __init__(self
, name
, module
, klass
=None):
26 # nested is true if the class could contain free variables,
27 # i.e. if it is nested within another function.
32 for i
in range(len(klass
)):
34 self
.klass
= klass
[i
:]
38 return "<%s: %s>" % (self
.__class
__.__name
__, self
.name
)
40 def mangle(self
, name
):
41 if self
.klass
is None:
43 return mangle(name
, self
.klass
)
45 def add_def(self
, name
):
46 self
.defs
[self
.mangle(name
)] = 1
48 def add_use(self
, name
):
49 self
.uses
[self
.mangle(name
)] = 1
51 def add_global(self
, name
):
52 name
= self
.mangle(name
)
53 if name
in self
.uses
or name
in self
.defs
:
54 pass # XXX warn about global following def/use
55 if name
in self
.params
:
56 raise SyntaxError, "%s in %s is global and parameter" % \
58 self
.globals[name
] = 1
59 self
.module
.add_def(name
)
61 def add_param(self
, name
):
62 name
= self
.mangle(name
)
70 d
.update(self
.globals)
73 def add_child(self
, child
):
74 self
.children
.append(child
)
76 def get_children(self
):
80 print >> sys
.stderr
, self
.name
, self
.nested
and "nested" or ""
81 print >> sys
.stderr
, "\tglobals: ", self
.globals
82 print >> sys
.stderr
, "\tcells: ", self
.cells
83 print >> sys
.stderr
, "\tdefs: ", self
.defs
84 print >> sys
.stderr
, "\tuses: ", self
.uses
85 print >> sys
.stderr
, "\tfrees:", self
.frees
87 def check_name(self
, name
):
88 """Return scope of name.
90 The scope of a name could be LOCAL, GLOBAL, FREE, or CELL.
92 if name
in self
.globals:
93 return SC_GLOBAL_EXPLICT
94 if name
in self
.cells
:
98 if self
.nested
and (name
in self
.frees
or name
in self
.uses
):
103 return SC_GLOBAL_IMPLICIT
105 def get_free_vars(self
):
109 free
.update(self
.frees
)
110 for name
in self
.uses
.keys():
111 if name
not in self
.defs
and name
not in self
.globals:
115 def handle_children(self
):
116 for child
in self
.children
:
117 frees
= child
.get_free_vars()
118 globals = self
.add_frees(frees
)
120 child
.force_global(name
)
122 def force_global(self
, name
):
123 """Force name to be global in scope.
125 Some child of the current node had a free reference to name.
126 When the child was processed, it was labelled a free
127 variable. Now that all its enclosing scope have been
128 processed, the name is known to be a global or builtin. So
129 walk back down the child chain and set the name to be global
132 Be careful to stop if a child does not think the name is
135 self
.globals[name
] = 1
136 if name
in self
.frees
:
138 for child
in self
.children
:
139 if child
.check_name(name
) == SC_FREE
:
140 child
.force_global(name
)
142 def add_frees(self
, names
):
143 """Process list of free vars from nested scope.
145 Returns a list of names that are either 1) declared global in the
146 parent or 2) undefined in a top-level parent. In either case,
147 the nested scope should treat them as globals.
151 sc
= self
.check_name(name
)
153 if sc
== SC_UNKNOWN
or sc
== SC_FREE \
154 or isinstance(self
, ClassScope
):
156 elif sc
== SC_GLOBAL_IMPLICIT
:
157 child_globals
.append(name
)
158 elif isinstance(self
, FunctionScope
) and sc
== SC_LOCAL
:
161 child_globals
.append(name
)
166 child_globals
.append(name
)
169 def get_cell_vars(self
):
170 return self
.cells
.keys()
172 class ModuleScope(Scope
):
173 __super_init
= Scope
.__init
__
176 self
.__super
_init
("global", self
)
178 class FunctionScope(Scope
):
181 class GenExprScope(Scope
):
182 __super_init
= Scope
.__init
__
186 def __init__(self
, module
, klass
=None):
189 self
.__super
_init
("generator expression<%d>"%i, module
, klass
)
193 keys
= Scope
.get_names(self
)
196 class LambdaScope(FunctionScope
):
197 __super_init
= Scope
.__init
__
201 def __init__(self
, module
, klass
=None):
204 self
.__super
_init
("lambda.%d" % i
, module
, klass
)
206 class ClassScope(Scope
):
207 __super_init
= Scope
.__init
__
209 def __init__(self
, name
, module
):
210 self
.__super
_init
(name
, module
, name
)
217 # node that define new scopes
219 def visitModule(self
, node
):
220 scope
= self
.module
= self
.scopes
[node
] = ModuleScope()
221 self
.visit(node
.node
, scope
)
223 visitExpression
= visitModule
225 def visitFunction(self
, node
, parent
):
227 self
.visit(node
.decorators
, parent
)
228 parent
.add_def(node
.name
)
229 for n
in node
.defaults
:
230 self
.visit(n
, parent
)
231 scope
= FunctionScope(node
.name
, self
.module
, self
.klass
)
232 if parent
.nested
or isinstance(parent
, FunctionScope
):
234 self
.scopes
[node
] = scope
235 self
._do
_args
(scope
, node
.argnames
)
236 self
.visit(node
.code
, scope
)
237 self
.handle_free_vars(scope
, parent
)
239 def visitGenExpr(self
, node
, parent
):
240 scope
= GenExprScope(self
.module
, self
.klass
);
241 if parent
.nested
or isinstance(parent
, FunctionScope
) \
242 or isinstance(parent
, GenExprScope
):
245 self
.scopes
[node
] = scope
246 self
.visit(node
.code
, scope
)
248 self
.handle_free_vars(scope
, parent
)
250 def visitGenExprInner(self
, node
, scope
):
251 for genfor
in node
.quals
:
252 self
.visit(genfor
, scope
)
254 self
.visit(node
.expr
, scope
)
256 def visitGenExprFor(self
, node
, scope
):
257 self
.visit(node
.assign
, scope
, 1)
258 self
.visit(node
.iter, scope
)
260 self
.visit(if_
, scope
)
262 def visitGenExprIf(self
, node
, scope
):
263 self
.visit(node
.test
, scope
)
265 def visitLambda(self
, node
, parent
, assign
=0):
266 # Lambda is an expression, so it could appear in an expression
267 # context where assign is passed. The transformer should catch
268 # any code that has a lambda on the left-hand side.
271 for n
in node
.defaults
:
272 self
.visit(n
, parent
)
273 scope
= LambdaScope(self
.module
, self
.klass
)
274 if parent
.nested
or isinstance(parent
, FunctionScope
):
276 self
.scopes
[node
] = scope
277 self
._do
_args
(scope
, node
.argnames
)
278 self
.visit(node
.code
, scope
)
279 self
.handle_free_vars(scope
, parent
)
281 def _do_args(self
, scope
, args
):
283 if type(name
) == types
.TupleType
:
284 self
._do
_args
(scope
, name
)
286 scope
.add_param(name
)
288 def handle_free_vars(self
, scope
, parent
):
289 parent
.add_child(scope
)
290 scope
.handle_children()
292 def visitClass(self
, node
, parent
):
293 parent
.add_def(node
.name
)
295 self
.visit(n
, parent
)
296 scope
= ClassScope(node
.name
, self
.module
)
297 if parent
.nested
or isinstance(parent
, FunctionScope
):
299 if node
.doc
is not None:
300 scope
.add_def('__doc__')
301 scope
.add_def('__module__')
302 self
.scopes
[node
] = scope
304 self
.klass
= node
.name
305 self
.visit(node
.code
, scope
)
307 self
.handle_free_vars(scope
, parent
)
309 # name can be a def or a use
311 # XXX a few calls and nodes expect a third "assign" arg that is
312 # true if the name is being used as an assignment. only
313 # expressions contained within statements may have the assign arg.
315 def visitName(self
, node
, scope
, assign
=0):
317 scope
.add_def(node
.name
)
319 scope
.add_use(node
.name
)
321 # operations that bind new names
323 def visitFor(self
, node
, scope
):
324 self
.visit(node
.assign
, scope
, 1)
325 self
.visit(node
.list, scope
)
326 self
.visit(node
.body
, scope
)
328 self
.visit(node
.else_
, scope
)
330 def visitFrom(self
, node
, scope
):
331 for name
, asname
in node
.names
:
334 scope
.add_def(asname
or name
)
336 def visitImport(self
, node
, scope
):
337 for name
, asname
in node
.names
:
341 scope
.add_def(asname
or name
)
343 def visitGlobal(self
, node
, scope
):
344 for name
in node
.names
:
345 scope
.add_global(name
)
347 def visitAssign(self
, node
, scope
):
348 """Propagate assignment flag down to child nodes.
350 The Assign node doesn't itself contains the variables being
351 assigned to. Instead, the children in node.nodes are visited
352 with the assign flag set to true. When the names occur in
353 those nodes, they are marked as defs.
355 Some names that occur in an assignment target are not bound by
356 the assignment, e.g. a name occurring inside a slice. The
357 visitor handles these nodes specially; they do not propagate
358 the assign flag to their children.
361 self
.visit(n
, scope
, 1)
362 self
.visit(node
.expr
, scope
)
364 def visitAssName(self
, node
, scope
, assign
=1):
365 scope
.add_def(node
.name
)
367 def visitAssAttr(self
, node
, scope
, assign
=0):
368 self
.visit(node
.expr
, scope
, 0)
370 def visitSubscript(self
, node
, scope
, assign
=0):
371 self
.visit(node
.expr
, scope
, 0)
373 self
.visit(n
, scope
, 0)
375 def visitSlice(self
, node
, scope
, assign
=0):
376 self
.visit(node
.expr
, scope
, 0)
378 self
.visit(node
.lower
, scope
, 0)
380 self
.visit(node
.upper
, scope
, 0)
382 def visitAugAssign(self
, node
, scope
):
383 # If the LHS is a name, then this counts as assignment.
384 # Otherwise, it's just use.
385 self
.visit(node
.node
, scope
)
386 if isinstance(node
.node
, ast
.Name
):
387 self
.visit(node
.node
, scope
, 1) # XXX worry about this
388 self
.visit(node
.expr
, scope
)
390 # prune if statements if tests are false
392 _const_types
= types
.StringType
, types
.IntType
, types
.FloatType
394 def visitIf(self
, node
, scope
):
395 for test
, body
in node
.tests
:
396 if isinstance(test
, ast
.Const
):
397 if type(test
.value
) in self
._const
_types
:
400 self
.visit(test
, scope
)
401 self
.visit(body
, scope
)
403 self
.visit(node
.else_
, scope
)
405 # a yield statement signals a generator
407 def visitYield(self
, node
, scope
):
409 self
.visit(node
.value
, scope
)
412 return sorted(l1
) == sorted(l2
)
414 if __name__
== "__main__":
416 from compiler
import parseFile
, walk
420 return [s
for s
in [s
.get_name() for s
in syms
.get_symbols()]
421 if not (s
.startswith('_[') or s
.startswith('.'))]
423 for file in sys
.argv
[1:]:
428 syms
= symtable
.symtable(buf
, file, "exec")
429 mod_names
= get_names(syms
)
430 tree
= parseFile(file)
434 # compare module-level symbols
435 names2
= s
.scopes
[tree
].get_names()
437 if not list_eq(mod_names
, names2
):
440 print sorted(mod_names
)
450 for s
in syms
.get_symbols():
452 l
= [sc
for sc
in scopes
453 if sc
.name
== s
.get_name()]
455 print "skipping", s
.get_name()
457 if not list_eq(get_names(s
.get_namespace()),
460 print sorted(get_names(s
.get_namespace()))
461 print sorted(l
[0].get_names())