Lib/pkgutil.py

   1 """Utilities to support packages."""
   2
   3 # NOTE: This module must remain compatible with Python 2.3, as it is shared
   4 # by setuptools for distribution with Python 2.3 and up.
   5
   6 import os
   7 import sys
   8 import imp
   9 import os.path
  10 from types import ModuleType
  11
  12 __all__ = [
  13     'get_importer', 'iter_importers', 'get_loader', 'find_loader',
  14     'walk_packages', 'iter_modules',
  15     'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
  16 ]
  17
  18 def read_code(stream):
  19     # This helper is needed in order for the PEP 302 emulation to
  20     # correctly handle compiled files
  21     import marshal
  22
  23     magic = stream.read(4)
  24     if magic != imp.get_magic():
  25         return None
  26
  27     stream.read(4) # Skip timestamp
  28     return marshal.load(stream)
  29
  30
  31 def simplegeneric(func):
  32     """Make a trivial single-dispatch generic function"""
  33     registry = {}
  34     def wrapper(*args, **kw):
  35         ob = args[0]
  36         try:
  37             cls = ob.__class__
  38         except AttributeError:
  39             cls = type(ob)
  40         try:
  41             mro = cls.__mro__
  42         except AttributeError:
  43             try:
  44                 class cls(cls, object):
  45                     pass
  46                 mro = cls.__mro__[1:]
  47             except TypeError:
  48                 mro = object,   # must be an ExtensionClass or some such  :(
  49         for t in mro:
  50             if t in registry:
  51                 return registry[t](*args, **kw)
  52         else:
  53             return func(*args, **kw)
  54     try:
  55         wrapper.__name__ = func.__name__
  56     except (TypeError, AttributeError):
  57         pass    # Python 2.3 doesn't allow functions to be renamed
  58
  59     def register(typ, func=None):
  60         if func is None:
  61             return lambda f: register(typ, f)
  62         registry[typ] = func
  63         return func
  64
  65     wrapper.__dict__ = func.__dict__
  66     wrapper.__doc__ = func.__doc__
  67     wrapper.register = register
  68     return wrapper
  69
  70
  71 def walk_packages(path=None, prefix='', onerror=None):
  72     """Yields (module_loader, name, ispkg) for all modules recursively
  73     on path, or, if path is None, all accessible modules.
  74
  75     'path' should be either None or a list of paths to look for
  76     modules in.
  77
  78     'prefix' is a string to output on the front of every module name
  79     on output.
  80
  81     Note that this function must import all *packages* (NOT all
  82     modules!) on the given path, in order to access the __path__
  83     attribute to find submodules.
  84
  85     'onerror' is a function which gets called with one argument (the
  86     name of the package which was being imported) if any exception
  87     occurs while trying to import a package.  If no onerror function is
  88     supplied, ImportErrors are caught and ignored, while all other
  89     exceptions are propagated, terminating the search.
  90
  91     Examples:
  92
  93     # list all modules python can access
  94     walk_packages()
  95
  96     # list all submodules of ctypes
  97     walk_packages(ctypes.__path__, ctypes.__name__+'.')
  98     """
  99
 100     def seen(p, m={}):
 101         if p in m:
 102             return True
 103         m[p] = True
 104
 105     for importer, name, ispkg in iter_modules(path, prefix):
 106         yield importer, name, ispkg
 107
 108         if ispkg:
 109             try:
 110                 __import__(name)
 111             except ImportError:
 112                 if onerror is not None:
 113                     onerror(name)
 114             except Exception:
 115                 if onerror is not None:
 116                     onerror(name)
 117                 else:
 118                     raise
 119             else:
 120                 path = getattr(sys.modules[name], '__path__', None) or []
 121
 122                 # don't traverse path items we've seen before
 123                 path = [p for p in path if not seen(p)]
 124
 125                 for item in walk_packages(path, name+'.', onerror):
 126                     yield item
 127
 128
 129 def iter_modules(path=None, prefix=''):
 130     """Yields (module_loader, name, ispkg) for all submodules on path,
 131     or, if path is None, all top-level modules on sys.path.
 132
 133     'path' should be either None or a list of paths to look for
 134     modules in.
 135
 136     'prefix' is a string to output on the front of every module name
 137     on output.
 138     """
 139
 140     if path is None:
 141         importers = iter_importers()
 142     else:
 143         importers = map(get_importer, path)
 144
 145     yielded = {}
 146     for i in importers:
 147         for name, ispkg in iter_importer_modules(i, prefix):
 148             if name not in yielded:
 149                 yielded[name] = 1
 150                 yield i, name, ispkg
 151
 152
 153 #@simplegeneric
 154 def iter_importer_modules(importer, prefix=''):
 155     if not hasattr(importer, 'iter_modules'):
 156         return []
 157     return importer.iter_modules(prefix)
 158
 159 iter_importer_modules = simplegeneric(iter_importer_modules)
 160
 161
 162 class ImpImporter:
 163     """PEP 302 Importer that wraps Python's "classic" import algorithm
 164
 165     ImpImporter(dirname) produces a PEP 302 importer that searches that
 166     directory.  ImpImporter(None) produces a PEP 302 importer that searches
 167     the current sys.path, plus any modules that are frozen or built-in.
 168
 169     Note that ImpImporter does not currently support being used by placement
 170     on sys.meta_path.
 171     """
 172
 173     def __init__(self, path=None):
 174         self.path = path
 175
 176     def find_module(self, fullname, path=None):
 177         # Note: we ignore 'path' argument since it is only used via meta_path
 178         subname = fullname.split(".")[-1]
 179         if subname != fullname and self.path is None:
 180             return None
 181         if self.path is None:
 182             path = None
 183         else:
 184             path = [os.path.realpath(self.path)]
 185         try:
 186             file, filename, etc = imp.find_module(subname, path)
 187         except ImportError:
 188             return None
 189         return ImpLoader(fullname, file, filename, etc)
 190
 191     def iter_modules(self, prefix=''):
 192         if self.path is None or not os.path.isdir(self.path):
 193             return
 194
 195         yielded = {}
 196         import inspect
 197
 198         filenames = os.listdir(self.path)
 199         filenames.sort()  # handle packages before same-named modules
 200
 201         for fn in filenames:
 202             modname = inspect.getmodulename(fn)
 203             if modname=='__init__' or modname in yielded:
 204                 continue
 205
 206             path = os.path.join(self.path, fn)
 207             ispkg = False
 208
 209             if not modname and os.path.isdir(path) and '.' not in fn:
 210                 modname = fn
 211                 for fn in os.listdir(path):
 212                     subname = inspect.getmodulename(fn)
 213                     if subname=='__init__':
 214                         ispkg = True
 215                         break
 216                 else:
 217                     continue    # not a package
 218
 219             if modname and '.' not in modname:
 220                 yielded[modname] = 1
 221                 yield prefix + modname, ispkg
 222
 223
 224 class ImpLoader:
 225     """PEP 302 Loader that wraps Python's "classic" import algorithm
 226     """
 227     code = source = None
 228
 229     def __init__(self, fullname, file, filename, etc):
 230         self.file = file
 231         self.filename = filename
 232         self.fullname = fullname
 233         self.etc = etc
 234
 235     def load_module(self, fullname):
 236         self._reopen()
 237         try:
 238             mod = imp.load_module(fullname, self.file, self.filename, self.etc)
 239         finally:
 240             if self.file:
 241                 self.file.close()
 242         # Note: we don't set __loader__ because we want the module to look
 243         # normal; i.e. this is just a wrapper for standard import machinery
 244         return mod
 245
 246     def get_data(self, pathname):
 247         return open(pathname, "rb").read()
 248
 249     def _reopen(self):
 250         if self.file and self.file.closed:
 251             mod_type = self.etc[2]
 252             if mod_type==imp.PY_SOURCE:
 253                 self.file = open(self.filename, 'rU')
 254             elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
 255                 self.file = open(self.filename, 'rb')
 256
 257     def _fix_name(self, fullname):
 258         if fullname is None:
 259             fullname = self.fullname
 260         elif fullname != self.fullname:
 261             raise ImportError("Loader for module %s cannot handle "
 262                               "module %s" % (self.fullname, fullname))
 263         return fullname
 264
 265     def is_package(self, fullname):
 266         fullname = self._fix_name(fullname)
 267         return self.etc[2]==imp.PKG_DIRECTORY
 268
 269     def get_code(self, fullname=None):
 270         fullname = self._fix_name(fullname)
 271         if self.code is None:
 272             mod_type = self.etc[2]
 273             if mod_type==imp.PY_SOURCE:
 274                 source = self.get_source(fullname)
 275                 self.code = compile(source, self.filename, 'exec')
 276             elif mod_type==imp.PY_COMPILED:
 277                 self._reopen()
 278                 try:
 279                     self.code = read_code(self.file)
 280                 finally:
 281                     self.file.close()
 282             elif mod_type==imp.PKG_DIRECTORY:
 283                 self.code = self._get_delegate().get_code()
 284         return self.code
 285
 286     def get_source(self, fullname=None):
 287         fullname = self._fix_name(fullname)
 288         if self.source is None:
 289             mod_type = self.etc[2]
 290             if mod_type==imp.PY_SOURCE:
 291                 self._reopen()
 292                 try:
 293                     self.source = self.file.read()
 294                 finally:
 295                     self.file.close()
 296             elif mod_type==imp.PY_COMPILED:
 297                 if os.path.exists(self.filename[:-1]):
 298                     f = open(self.filename[:-1], 'rU')
 299                     self.source = f.read()
 300                     f.close()
 301             elif mod_type==imp.PKG_DIRECTORY:
 302                 self.source = self._get_delegate().get_source()
 303         return self.source
 304
 305
 306     def _get_delegate(self):
 307         return ImpImporter(self.filename).find_module('__init__')
 308
 309     def get_filename(self, fullname=None):
 310         fullname = self._fix_name(fullname)
 311         mod_type = self.etc[2]
 312         if self.etc[2]==imp.PKG_DIRECTORY:
 313             return self._get_delegate().get_filename()
 314         elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
 315             return self.filename
 316         return None
 317
 318
 319 try:
 320     import zipimport
 321     from zipimport import zipimporter
 322
 323     def iter_zipimport_modules(importer, prefix=''):
 324         dirlist = zipimport._zip_directory_cache[importer.archive].keys()
 325         dirlist.sort()
 326         _prefix = importer.prefix
 327         plen = len(_prefix)
 328         yielded = {}
 329         import inspect
 330         for fn in dirlist:
 331             if not fn.startswith(_prefix):
 332                 continue
 333
 334             fn = fn[plen:].split(os.sep)
 335
 336             if len(fn)==2 and fn[1].startswith('__init__.py'):
 337                 if fn[0] not in yielded:
 338                     yielded[fn[0]] = 1
 339                     yield fn[0], True
 340
 341             if len(fn)!=1:
 342                 continue
 343
 344             modname = inspect.getmodulename(fn[0])
 345             if modname=='__init__':
 346                 continue
 347
 348             if modname and '.' not in modname and modname not in yielded:
 349                 yielded[modname] = 1
 350                 yield prefix + modname, False
 351
 352     iter_importer_modules.register(zipimporter, iter_zipimport_modules)
 353
 354 except ImportError:
 355     pass
 356
 357
 358 def get_importer(path_item):
 359     """Retrieve a PEP 302 importer for the given path item
 360
 361     The returned importer is cached in sys.path_importer_cache
 362     if it was newly created by a path hook.
 363
 364     If there is no importer, a wrapper around the basic import
 365     machinery is returned. This wrapper is never inserted into
 366     the importer cache (None is inserted instead).
 367
 368     The cache (or part of it) can be cleared manually if a
 369     rescan of sys.path_hooks is necessary.
 370     """
 371     try:
 372         importer = sys.path_importer_cache[path_item]
 373     except KeyError:
 374         for path_hook in sys.path_hooks:
 375             try:
 376                 importer = path_hook(path_item)
 377                 break
 378             except ImportError:
 379                 pass
 380         else:
 381             importer = None
 382         sys.path_importer_cache.setdefault(path_item, importer)
 383
 384     if importer is None:
 385         try:
 386             importer = ImpImporter(path_item)
 387         except ImportError:
 388             importer = None
 389     return importer
 390
 391
 392 def iter_importers(fullname=""):
 393     """Yield PEP 302 importers for the given module name
 394
 395     If fullname contains a '.', the importers will be for the package
 396     containing fullname, otherwise they will be importers for sys.meta_path,
 397     sys.path, and Python's "classic" import machinery, in that order.  If
 398     the named module is in a package, that package is imported as a side
 399     effect of invoking this function.
 400
 401     Non PEP 302 mechanisms (e.g. the Windows registry) used by the
 402     standard import machinery to find files in alternative locations
 403     are partially supported, but are searched AFTER sys.path. Normally,
 404     these locations are searched BEFORE sys.path, preventing sys.path
 405     entries from shadowing them.
 406
 407     For this to cause a visible difference in behaviour, there must
 408     be a module or package name that is accessible via both sys.path
 409     and one of the non PEP 302 file system mechanisms. In this case,
 410     the emulation will find the former version, while the builtin
 411     import mechanism will find the latter.
 412
 413     Items of the following types can be affected by this discrepancy:
 414         imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
 415     """
 416     if fullname.startswith('.'):
 417         raise ImportError("Relative module names not supported")
 418     if '.' in fullname:
 419         # Get the containing package's __path__
 420         pkg = '.'.join(fullname.split('.')[:-1])
 421         if pkg not in sys.modules:
 422             __import__(pkg)
 423         path = getattr(sys.modules[pkg], '__path__', None) or []
 424     else:
 425         for importer in sys.meta_path:
 426             yield importer
 427         path = sys.path
 428     for item in path:
 429         yield get_importer(item)
 430     if '.' not in fullname:
 431         yield ImpImporter()
 432
 433 def get_loader(module_or_name):
 434     """Get a PEP 302 "loader" object for module_or_name
 435
 436     If the module or package is accessible via the normal import
 437     mechanism, a wrapper around the relevant part of that machinery
 438     is returned.  Returns None if the module cannot be found or imported.
 439     If the named module is not already imported, its containing package
 440     (if any) is imported, in order to establish the package __path__.
 441
 442     This function uses iter_importers(), and is thus subject to the same
 443     limitations regarding platform-specific special import locations such
 444     as the Windows registry.
 445     """
 446     if module_or_name in sys.modules:
 447         module_or_name = sys.modules[module_or_name]
 448     if isinstance(module_or_name, ModuleType):
 449         module = module_or_name
 450         loader = getattr(module, '__loader__', None)
 451         if loader is not None:
 452             return loader
 453         fullname = module.__name__
 454     else:
 455         fullname = module_or_name
 456     return find_loader(fullname)
 457
 458 def find_loader(fullname):
 459     """Find a PEP 302 "loader" object for fullname
 460
 461     If fullname contains dots, path must be the containing package's __path__.
 462     Returns None if the module cannot be found or imported. This function uses
 463     iter_importers(), and is thus subject to the same limitations regarding
 464     platform-specific special import locations such as the Windows registry.
 465     """
 466     for importer in iter_importers(fullname):
 467         loader = importer.find_module(fullname)
 468         if loader is not None:
 469             return loader
 470
 471     return None
 472
 473
 474 def extend_path(path, name):
 475     """Extend a package's path.
 476
 477     Intended use is to place the following code in a package's __init__.py:
 478
 479         from pkgutil import extend_path
 480         __path__ = extend_path(__path__, __name__)
 481
 482     This will add to the package's __path__ all subdirectories of
 483     directories on sys.path named after the package.  This is useful
 484     if one wants to distribute different parts of a single logical
 485     package as multiple directories.
 486
 487     It also looks for *.pkg files beginning where * matches the name
 488     argument.  This feature is similar to *.pth files (see site.py),
 489     except that it doesn't special-case lines starting with 'import'.
 490     A *.pkg file is trusted at face value: apart from checking for
 491     duplicates, all entries found in a *.pkg file are added to the
 492     path, regardless of whether they are exist the filesystem.  (This
 493     is a feature.)
 494
 495     If the input path is not a list (as is the case for frozen
 496     packages) it is returned unchanged.  The input path is not
 497     modified; an extended copy is returned.  Items are only appended
 498     to the copy at the end.
 499
 500     It is assumed that sys.path is a sequence.  Items of sys.path that
 501     are not (unicode or 8-bit) strings referring to existing
 502     directories are ignored.  Unicode items of sys.path that cause
 503     errors when used as filenames may cause this function to raise an
 504     exception (in line with os.path.isdir() behavior).
 505     """
 506
 507     if not isinstance(path, list):
 508         # This could happen e.g. when this is called from inside a
 509         # frozen package.  Return the path unchanged in that case.
 510         return path
 511
 512     pname = os.path.join(*name.split('.')) # Reconstitute as relative path
 513     # Just in case os.extsep != '.'
 514     sname = os.extsep.join(name.split('.'))
 515     sname_pkg = sname + os.extsep + "pkg"
 516     init_py = "__init__" + os.extsep + "py"
 517
 518     path = path[:] # Start with a copy of the existing path
 519
 520     for dir in sys.path:
 521         if not isinstance(dir, basestring) or not os.path.isdir(dir):
 522             continue
 523         subdir = os.path.join(dir, pname)
 524         # XXX This may still add duplicate entries to path on
 525         # case-insensitive filesystems
 526         initfile = os.path.join(subdir, init_py)
 527         if subdir not in path and os.path.isfile(initfile):
 528             path.append(subdir)
 529         # XXX Is this the right thing for subpackages like zope.app?
 530         # It looks for a file named "zope.app.pkg"
 531         pkgfile = os.path.join(dir, sname_pkg)
 532         if os.path.isfile(pkgfile):
 533             try:
 534                 f = open(pkgfile)
 535             except IOError, msg:
 536                 sys.stderr.write("Can't open %s: %s\n" %
 537                                  (pkgfile, msg))
 538             else:
 539                 for line in f:
 540                     line = line.rstrip('\n')
 541                     if not line or line.startswith('#'):
 542                         continue
 543                     path.append(line) # Don't check for existence!
 544                 f.close()
 545
 546     return path
 547
 548 def get_data(package, resource):
 549     """Get a resource from a package.
 550
 551     This is a wrapper round the PEP 302 loader get_data API. The package
 552     argument should be the name of a package, in standard module format
 553     (foo.bar). The resource argument should be in the form of a relative
 554     filename, using '/' as the path separator. The parent directory name '..'
 555     is not allowed, and nor is a rooted name (starting with a '/').
 556
 557     The function returns a binary string, which is the contents of the
 558     specified resource.
 559
 560     For packages located in the filesystem, which have already been imported,
 561     this is the rough equivalent of
 562
 563         d = os.path.dirname(sys.modules[package].__file__)
 564         data = open(os.path.join(d, resource), 'rb').read()
 565
 566     If the package cannot be located or loaded, or it uses a PEP 302 loader
 567     which does not support get_data(), then None is returned.
 568     """
 569
 570     loader = get_loader(package)
 571     if loader is None or not hasattr(loader, 'get_data'):
 572         return None
 573     mod = sys.modules.get(package) or loader.load_module(package)
 574     if mod is None or not hasattr(mod, '__file__'):
 575         return None
 576
 577     # Modify the resource name to be compatible with the loader.get_data
 578     # signature - an os.path format "filename" starting with the dirname of
 579     # the package's __file__
 580     parts = resource.split('/')
 581     parts.insert(0, os.path.dirname(mod.__file__))
 582     resource_name = os.path.join(*parts)
 583     return loader.get_data(resource_name)