1 """Find modules used by a script, using introspection."""
2 # This module should be kept compatible with Python 2.2, see PEP 291.
4 from __future__
import generators
13 if hasattr(sys
.__stdout
__, "newlines"):
14 READ_MODE
= "U" # universal line endings
16 # remain compatible with Python < 2.3
19 LOAD_CONST
= chr(dis
.opname
.index('LOAD_CONST'))
20 IMPORT_NAME
= chr(dis
.opname
.index('IMPORT_NAME'))
21 STORE_NAME
= chr(dis
.opname
.index('STORE_NAME'))
22 STORE_GLOBAL
= chr(dis
.opname
.index('STORE_GLOBAL'))
23 STORE_OPS
= [STORE_NAME
, STORE_GLOBAL
]
24 HAVE_ARGUMENT
= chr(dis
.HAVE_ARGUMENT
)
26 # Modulefinder does a good job at simulating Python's, but it can not
27 # handle __path__ modifications packages make at runtime. Therefore there
28 # is a mechanism whereby you can register extra paths in this map for a
29 # package, and it will be honored.
31 # Note this is a mapping is lists of paths.
35 def AddPackagePath(packagename
, path
):
36 paths
= packagePathMap
.get(packagename
, [])
38 packagePathMap
[packagename
] = paths
40 replacePackageMap
= {}
42 # This ReplacePackage mechanism allows modulefinder to work around the
43 # way the _xmlplus package injects itself under the name "xml" into
44 # sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml")
45 # before running ModuleFinder.
47 def ReplacePackage(oldname
, newname
):
48 replacePackageMap
[oldname
] = newname
53 def __init__(self
, name
, file=None, path
=None):
58 # The set of global names that are assigned to in the module.
59 # This includes those names imported through starimports of
62 # The set of starimports this module did that could not be
63 # resolved, ie. a starimport from a non-Python module.
67 s
= "Module(%r" % (self
.__name
__,)
68 if self
.__file
__ is not None:
69 s
= s
+ ", %r" % (self
.__file
__,)
70 if self
.__path
__ is not None:
71 s
= s
+ ", %r" % (self
.__path
__,)
77 def __init__(self
, path
=None, debug
=0, excludes
=[], replace_paths
=[]):
85 self
.excludes
= excludes
86 self
.replace_paths
= replace_paths
87 self
.processed_paths
= [] # Used in debugging only
89 def msg(self
, level
, str, *args
):
90 if level
<= self
.debug
:
91 for i
in range(self
.indent
):
98 def msgin(self
, *args
):
100 if level
<= self
.debug
:
101 self
.indent
= self
.indent
+ 1
104 def msgout(self
, *args
):
106 if level
<= self
.debug
:
107 self
.indent
= self
.indent
- 1
110 def run_script(self
, pathname
):
111 self
.msg(2, "run_script", pathname
)
112 fp
= open(pathname
, READ_MODE
)
113 stuff
= ("", "r", imp
.PY_SOURCE
)
114 self
.load_module('__main__', fp
, pathname
, stuff
)
116 def load_file(self
, pathname
):
117 dir, name
= os
.path
.split(pathname
)
118 name
, ext
= os
.path
.splitext(name
)
119 fp
= open(pathname
, READ_MODE
)
120 stuff
= (ext
, "r", imp
.PY_SOURCE
)
121 self
.load_module(name
, fp
, pathname
, stuff
)
123 def import_hook(self
, name
, caller
=None, fromlist
=None, level
=-1):
124 self
.msg(3, "import_hook", name
, caller
, fromlist
, level
)
125 parent
= self
.determine_parent(caller
, level
=level
)
126 q
, tail
= self
.find_head_package(parent
, name
)
127 m
= self
.load_tail(q
, tail
)
131 self
.ensure_fromlist(m
, fromlist
)
134 def determine_parent(self
, caller
, level
=-1):
135 self
.msgin(4, "determine_parent", caller
, level
)
136 if not caller
or level
== 0:
137 self
.msgout(4, "determine_parent -> None")
139 pname
= caller
.__name
__
140 if level
>= 1: # relative import
144 parent
= self
.modules
[pname
]
145 assert parent
is caller
146 self
.msgout(4, "determine_parent ->", parent
)
148 if pname
.count(".") < level
:
149 raise ImportError, "relative importpath too deep"
150 pname
= ".".join(pname
.split(".")[:-level
])
151 parent
= self
.modules
[pname
]
152 self
.msgout(4, "determine_parent ->", parent
)
155 parent
= self
.modules
[pname
]
156 assert caller
is parent
157 self
.msgout(4, "determine_parent ->", parent
)
162 parent
= self
.modules
[pname
]
163 assert parent
.__name
__ == pname
164 self
.msgout(4, "determine_parent ->", parent
)
166 self
.msgout(4, "determine_parent -> None")
169 def find_head_package(self
, parent
, name
):
170 self
.msgin(4, "find_head_package", parent
, name
)
179 qname
= "%s.%s" % (parent
.__name
__, head
)
182 q
= self
.import_module(head
, qname
, parent
)
184 self
.msgout(4, "find_head_package ->", (q
, tail
))
189 q
= self
.import_module(head
, qname
, parent
)
191 self
.msgout(4, "find_head_package ->", (q
, tail
))
193 self
.msgout(4, "raise ImportError: No module named", qname
)
194 raise ImportError, "No module named " + qname
196 def load_tail(self
, q
, tail
):
197 self
.msgin(4, "load_tail", q
, tail
)
201 if i
< 0: i
= len(tail
)
202 head
, tail
= tail
[:i
], tail
[i
+1:]
203 mname
= "%s.%s" % (m
.__name
__, head
)
204 m
= self
.import_module(head
, mname
, m
)
206 self
.msgout(4, "raise ImportError: No module named", mname
)
207 raise ImportError, "No module named " + mname
208 self
.msgout(4, "load_tail ->", m
)
211 def ensure_fromlist(self
, m
, fromlist
, recursive
=0):
212 self
.msg(4, "ensure_fromlist", m
, fromlist
, recursive
)
216 all
= self
.find_all_submodules(m
)
218 self
.ensure_fromlist(m
, all
, 1)
219 elif not hasattr(m
, sub
):
220 subname
= "%s.%s" % (m
.__name
__, sub
)
221 submod
= self
.import_module(sub
, subname
, m
)
223 raise ImportError, "No module named " + subname
225 def find_all_submodules(self
, m
):
229 # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"].
230 # But we must also collect Python extension modules - although
231 # we cannot separate normal dlls from Python extensions.
233 for triple
in imp
.get_suffixes():
234 suffixes
.append(triple
[0])
235 for dir in m
.__path
__:
237 names
= os
.listdir(dir)
239 self
.msg(2, "can't list directory", dir)
243 for suff
in suffixes
:
245 if name
[-n
:] == suff
:
248 if mod
and mod
!= "__init__":
250 return modules
.keys()
252 def import_module(self
, partname
, fqname
, parent
):
253 self
.msgin(3, "import_module", partname
, fqname
, parent
)
255 m
= self
.modules
[fqname
]
259 self
.msgout(3, "import_module ->", m
)
261 if self
.badmodules
.has_key(fqname
):
262 self
.msgout(3, "import_module -> None")
264 if parent
and parent
.__path
__ is None:
265 self
.msgout(3, "import_module -> None")
268 fp
, pathname
, stuff
= self
.find_module(partname
,
269 parent
and parent
.__path
__, parent
)
271 self
.msgout(3, "import_module ->", None)
274 m
= self
.load_module(fqname
, fp
, pathname
, stuff
)
278 setattr(parent
, partname
, m
)
279 self
.msgout(3, "import_module ->", m
)
282 def load_module(self
, fqname
, fp
, pathname
, (suffix
, mode
, type)):
283 self
.msgin(2, "load_module", fqname
, fp
and "fp", pathname
)
284 if type == imp
.PKG_DIRECTORY
:
285 m
= self
.load_package(fqname
, pathname
)
286 self
.msgout(2, "load_module ->", m
)
288 if type == imp
.PY_SOURCE
:
289 co
= compile(fp
.read()+'\n', pathname
, 'exec')
290 elif type == imp
.PY_COMPILED
:
291 if fp
.read(4) != imp
.get_magic():
292 self
.msgout(2, "raise ImportError: Bad magic number", pathname
)
293 raise ImportError, "Bad magic number in %s" % pathname
295 co
= marshal
.load(fp
)
298 m
= self
.add_module(fqname
)
299 m
.__file
__ = pathname
301 if self
.replace_paths
:
302 co
= self
.replace_paths_in_code(co
)
304 self
.scan_code(co
, m
)
305 self
.msgout(2, "load_module ->", m
)
308 def _add_badmodule(self
, name
, caller
):
309 if name
not in self
.badmodules
:
310 self
.badmodules
[name
] = {}
311 self
.badmodules
[name
][caller
.__name
__] = 1
313 def _safe_import_hook(self
, name
, caller
, fromlist
, level
=-1):
314 # wrapper for self.import_hook() that won't raise ImportError
315 if name
in self
.badmodules
:
316 self
._add
_badmodule
(name
, caller
)
319 self
.import_hook(name
, caller
, level
=level
)
320 except ImportError, msg
:
321 self
.msg(2, "ImportError:", str(msg
))
322 self
._add
_badmodule
(name
, caller
)
326 if sub
in self
.badmodules
:
327 self
._add
_badmodule
(sub
, caller
)
330 self
.import_hook(name
, caller
, [sub
], level
=level
)
331 except ImportError, msg
:
332 self
.msg(2, "ImportError:", str(msg
))
333 fullname
= name
+ "." + sub
334 self
._add
_badmodule
(fullname
, caller
)
336 def scan_opcodes(self
, co
,
337 unpack
= struct
.unpack
):
338 # Scan the code, and yield 'interesting' opcode combinations
339 # Version for Python 2.4 and older
342 consts
= co
.co_consts
346 oparg
, = unpack('<H', code
[1:3])
347 yield "store", (names
[oparg
],)
350 if c
== LOAD_CONST
and code
[3] == IMPORT_NAME
:
351 oparg_1
, oparg_2
= unpack('<xHxH', code
[:6])
352 yield "import", (consts
[oparg_1
], names
[oparg_2
])
355 if c
>= HAVE_ARGUMENT
:
360 def scan_opcodes_25(self
, co
,
361 unpack
= struct
.unpack
):
362 # Scan the code, and yield 'interesting' opcode combinations
363 # Python 2.5 version (has absolute and relative imports)
366 consts
= co
.co_consts
367 LOAD_LOAD_AND_IMPORT
= LOAD_CONST
+ LOAD_CONST
+ IMPORT_NAME
371 oparg
, = unpack('<H', code
[1:3])
372 yield "store", (names
[oparg
],)
375 if code
[:9:3] == LOAD_LOAD_AND_IMPORT
:
376 oparg_1
, oparg_2
, oparg_3
= unpack('<xHxHxH', code
[:9])
377 level
= consts
[oparg_1
]
378 if level
== -1: # normal import
379 yield "import", (consts
[oparg_2
], names
[oparg_3
])
380 elif level
== 0: # absolute import
381 yield "absolute_import", (consts
[oparg_2
], names
[oparg_3
])
382 else: # relative import
383 yield "relative_import", (level
, consts
[oparg_2
], names
[oparg_3
])
386 if c
>= HAVE_ARGUMENT
:
391 def scan_code(self
, co
, m
):
393 if sys
.version_info
>= (2, 5):
394 scanner
= self
.scan_opcodes_25
396 scanner
= self
.scan_opcodes
397 for what
, args
in scanner(co
):
400 m
.globalnames
[name
] = 1
401 elif what
in ("import", "absolute_import"):
402 fromlist
, name
= args
404 if fromlist
is not None:
407 fromlist
= [f
for f
in fromlist
if f
!= "*"]
408 if what
== "absolute_import": level
= 0
410 self
._safe
_import
_hook
(name
, m
, fromlist
, level
=level
)
412 # We've encountered an "import *". If it is a Python module,
413 # the code has already been parsed and we can suck out the
417 # At this point we don't know whether 'name' is a
418 # submodule of 'm' or a global module. Let's just try
419 # the full name first.
420 mm
= self
.modules
.get(m
.__name
__ + "." + name
)
422 mm
= self
.modules
.get(name
)
424 m
.globalnames
.update(mm
.globalnames
)
425 m
.starimports
.update(mm
.starimports
)
426 if mm
.__code
__ is None:
427 m
.starimports
[name
] = 1
429 m
.starimports
[name
] = 1
430 elif what
== "relative_import":
431 level
, fromlist
, name
= args
433 self
._safe
_import
_hook
(name
, m
, fromlist
, level
=level
)
435 parent
= self
.determine_parent(m
, level
=level
)
436 self
._safe
_import
_hook
(parent
.__name
__, None, fromlist
, level
=0)
438 # We don't expect anything else from the generator.
439 raise RuntimeError(what
)
441 for c
in co
.co_consts
:
442 if isinstance(c
, type(co
)):
445 def load_package(self
, fqname
, pathname
):
446 self
.msgin(2, "load_package", fqname
, pathname
)
447 newname
= replacePackageMap
.get(fqname
)
450 m
= self
.add_module(fqname
)
451 m
.__file
__ = pathname
452 m
.__path
__ = [pathname
]
454 # As per comment at top of file, simulate runtime __path__ additions.
455 m
.__path
__ = m
.__path
__ + packagePathMap
.get(fqname
, [])
457 fp
, buf
, stuff
= self
.find_module("__init__", m
.__path
__)
458 self
.load_module(fqname
, fp
, buf
, stuff
)
459 self
.msgout(2, "load_package ->", m
)
462 def add_module(self
, fqname
):
463 if self
.modules
.has_key(fqname
):
464 return self
.modules
[fqname
]
465 self
.modules
[fqname
] = m
= Module(fqname
)
468 def find_module(self
, name
, path
, parent
=None):
469 if parent
is not None:
470 # assert path is not None
471 fullname
= parent
.__name
__+'.'+name
474 if fullname
in self
.excludes
:
475 self
.msgout(3, "find_module -> Excluded", fullname
)
476 raise ImportError, name
479 if name
in sys
.builtin_module_names
:
480 return (None, None, ("", "", imp
.C_BUILTIN
))
483 return imp
.find_module(name
, path
)
486 """Print a report to stdout, listing the found modules with their
487 paths, as well as modules that are missing, or seem to be missing.
490 print " %-25s %s" % ("Name", "File")
491 print " %-25s %s" % ("----", "----")
492 # Print modules found
493 keys
= self
.modules
.keys()
496 m
= self
.modules
[key
]
501 print "%-25s" % key
, m
.__file
__ or ""
503 # Print missing modules
504 missing
, maybe
= self
.any_missing_maybe()
507 print "Missing modules:"
509 mods
= self
.badmodules
[name
].keys()
511 print "?", name
, "imported from", ', '.join(mods
)
512 # Print modules that may be missing, but then again, maybe not...
515 print "Submodules thay appear to be missing, but could also be",
516 print "global names in the parent package:"
518 mods
= self
.badmodules
[name
].keys()
520 print "?", name
, "imported from", ', '.join(mods
)
522 def any_missing(self
):
523 """Return a list of modules that appear to be missing. Use
524 any_missing_maybe() if you want to know which modules are
525 certain to be missing, and which *may* be missing.
527 missing
, maybe
= self
.any_missing_maybe()
528 return missing
+ maybe
530 def any_missing_maybe(self
):
531 """Return two lists, one with modules that are certainly missing
532 and one with modules that *may* be missing. The latter names could
533 either be submodules *or* just global names in the package.
535 The reason it can't always be determined is that it's impossible to
536 tell which names are imported when "from module import *" is done
537 with an extension module, short of actually importing it.
541 for name
in self
.badmodules
:
542 if name
in self
.excludes
:
550 pkg
= self
.modules
.get(pkgname
)
552 if pkgname
in self
.badmodules
[name
]:
553 # The package tried to import this module itself and
554 # failed. It's definitely missing.
556 elif subname
in pkg
.globalnames
:
557 # It's a global in the package: definitely not missing.
559 elif pkg
.starimports
:
560 # It could be missing, but the package did an "import *"
561 # from a non-Python module, so we simply can't be sure.
564 # It's not a global in the package, the package didn't
565 # do funny star imports, it's very likely to be missing.
566 # The symbol could be inserted into the package from the
567 # outside, but since that's not good style we simply list
574 return missing
, maybe
576 def replace_paths_in_code(self
, co
):
577 new_filename
= original_filename
= os
.path
.normpath(co
.co_filename
)
578 for f
, r
in self
.replace_paths
:
579 if original_filename
.startswith(f
):
580 new_filename
= r
+ original_filename
[len(f
):]
583 if self
.debug
and original_filename
not in self
.processed_paths
:
584 if new_filename
!= original_filename
:
585 self
.msgout(2, "co_filename %r changed to %r" \
586 % (original_filename
,new_filename
,))
588 self
.msgout(2, "co_filename %r remains unchanged" \
589 % (original_filename
,))
590 self
.processed_paths
.append(original_filename
)
592 consts
= list(co
.co_consts
)
593 for i
in range(len(consts
)):
594 if isinstance(consts
[i
], type(co
)):
595 consts
[i
] = self
.replace_paths_in_code(consts
[i
])
597 return new
.code(co
.co_argcount
, co
.co_nlocals
, co
.co_stacksize
,
598 co
.co_flags
, co
.co_code
, tuple(consts
), co
.co_names
,
599 co
.co_varnames
, new_filename
, co
.co_name
,
600 co
.co_firstlineno
, co
.co_lnotab
,
601 co
.co_freevars
, co
.co_cellvars
)
608 opts
, args
= getopt
.getopt(sys
.argv
[1:], "dmp:qx:")
609 except getopt
.error
, msg
:
624 addpath
= addpath
+ a
.split(os
.pathsep
)
630 # Provide default arguments
636 # Set the path based on sys.path and the script directory
638 path
[0] = os
.path
.dirname(script
)
639 path
= addpath
+ path
643 print " ", repr(item
)
645 # Create the module finder and turn its crank
646 mf
= ModuleFinder(path
, debug
, exclude
)
653 mf
.import_hook(arg
[:-2], None, ["*"])
658 mf
.run_script(script
)
660 return mf
# for -i debugging
663 if __name__
== '__main__':
666 except KeyboardInterrupt:
667 print "\n[interrupt]"