Lib/pkgutil.py

   1 """Utilities to support packages."""
   2
   3 import os
   4 import sys
   5 import imp
   6 import os.path
   7 from types import ModuleType
   8
   9 __all__ = [
  10     'get_importer', 'iter_importers', 'get_loader', 'find_loader',
  11     'walk_packages', 'iter_modules',
  12     'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
  13 ]
  14
  15 def read_code(stream):
  16     # This helper is needed in order for the PEP 302 emulation to
  17     # correctly handle compiled files
  18     import marshal
  19
  20     magic = stream.read(4)
  21     if magic != imp.get_magic():
  22         return None
  23
  24     stream.read(4) # Skip timestamp
  25     return marshal.load(stream)
  26
  27
  28 def simplegeneric(func):
  29     """Make a trivial single-dispatch generic function"""
  30     registry = {}
  31     def wrapper(*args, **kw):
  32         ob = args[0]
  33         try:
  34             cls = ob.__class__
  35         except AttributeError:
  36             cls = type(ob)
  37         try:
  38             mro = cls.__mro__
  39         except AttributeError:
  40             try:
  41                 class cls(cls, object):
  42                     pass
  43                 mro = cls.__mro__[1:]
  44             except TypeError:
  45                 mro = object,   # must be an ExtensionClass or some such  :(
  46         for t in mro:
  47             if t in registry:
  48                 return registry[t](*args, **kw)
  49         else:
  50             return func(*args, **kw)
  51     try:
  52         wrapper.__name__ = func.__name__
  53     except (TypeError, AttributeError):
  54         pass    # Python 2.3 doesn't allow functions to be renamed
  55
  56     def register(typ, func=None):
  57         if func is None:
  58             return lambda f: register(typ, f)
  59         registry[typ] = func
  60         return func
  61
  62     wrapper.__dict__ = func.__dict__
  63     wrapper.__doc__ = func.__doc__
  64     wrapper.register = register
  65     return wrapper
  66
  67
  68 def walk_packages(path=None, prefix='', onerror=None):
  69     """Yields (module_loader, name, ispkg) for all modules recursively
  70     on path, or, if path is None, all accessible modules.
  71
  72     'path' should be either None or a list of paths to look for
  73     modules in.
  74
  75     'prefix' is a string to output on the front of every module name
  76     on output.
  77
  78     Note that this function must import all *packages* (NOT all
  79     modules!) on the given path, in order to access the __path__
  80     attribute to find submodules.
  81
  82     'onerror' is a function which gets called with one argument (the
  83     name of the package which was being imported) if any exception
  84     occurs while trying to import a package.  If no onerror function is
  85     supplied, ImportErrors are caught and ignored, while all other
  86     exceptions are propagated, terminating the search.
  87
  88     Examples:
  89
  90     # list all modules python can access
  91     walk_packages()
  92
  93     # list all submodules of ctypes
  94     walk_packages(ctypes.__path__, ctypes.__name__+'.')
  95     """
  96
  97     def seen(p, m={}):
  98         if p in m:
  99             return True
 100         m[p] = True
 101
 102     for importer, name, ispkg in iter_modules(path, prefix):
 103         yield importer, name, ispkg
 104
 105         if ispkg:
 106             try:
 107                 __import__(name)
 108             except ImportError:
 109                 if onerror is not None:
 110                     onerror(name)
 111             except Exception:
 112                 if onerror is not None:
 113                     onerror(name)
 114                 else:
 115                     raise
 116             else:
 117                 path = getattr(sys.modules[name], '__path__', None) or []
 118
 119                 # don't traverse path items we've seen before
 120                 path = [p for p in path if not seen(p)]
 121
 122                 for item in walk_packages(path, name+'.', onerror):
 123                     yield item
 124
 125
 126 def iter_modules(path=None, prefix=''):
 127     """Yields (module_loader, name, ispkg) for all submodules on path,
 128     or, if path is None, all top-level modules on sys.path.
 129
 130     'path' should be either None or a list of paths to look for
 131     modules in.
 132
 133     'prefix' is a string to output on the front of every module name
 134     on output.
 135     """
 136
 137     if path is None:
 138         importers = iter_importers()
 139     else:
 140         importers = map(get_importer, path)
 141
 142     yielded = {}
 143     for i in importers:
 144         for name, ispkg in iter_importer_modules(i, prefix):
 145             if name not in yielded:
 146                 yielded[name] = 1
 147                 yield i, name, ispkg
 148
 149
 150 #@simplegeneric
 151 def iter_importer_modules(importer, prefix=''):
 152     if not hasattr(importer, 'iter_modules'):
 153         return []
 154     return importer.iter_modules(prefix)
 155
 156 iter_importer_modules = simplegeneric(iter_importer_modules)
 157
 158
 159 class ImpImporter:
 160     """PEP 302 Importer that wraps Python's "classic" import algorithm
 161
 162     ImpImporter(dirname) produces a PEP 302 importer that searches that
 163     directory.  ImpImporter(None) produces a PEP 302 importer that searches
 164     the current sys.path, plus any modules that are frozen or built-in.
 165
 166     Note that ImpImporter does not currently support being used by placement
 167     on sys.meta_path.
 168     """
 169
 170     def __init__(self, path=None):
 171         self.path = path
 172
 173     def find_module(self, fullname, path=None):
 174         # Note: we ignore 'path' argument since it is only used via meta_path
 175         subname = fullname.split(".")[-1]
 176         if subname != fullname and self.path is None:
 177             return None
 178         if self.path is None:
 179             path = None
 180         else:
 181             path = [os.path.realpath(self.path)]
 182         try:
 183             file, filename, etc = imp.find_module(subname, path)
 184         except ImportError:
 185             return None
 186         return ImpLoader(fullname, file, filename, etc)
 187
 188     def iter_modules(self, prefix=''):
 189         if self.path is None or not os.path.isdir(self.path):
 190             return
 191
 192         yielded = {}
 193         import inspect
 194
 195         filenames = os.listdir(self.path)
 196         filenames.sort()  # handle packages before same-named modules
 197
 198         for fn in filenames:
 199             modname = inspect.getmodulename(fn)
 200             if modname=='__init__' or modname in yielded:
 201                 continue
 202
 203             path = os.path.join(self.path, fn)
 204             ispkg = False
 205
 206             if not modname and os.path.isdir(path) and '.' not in fn:
 207                 modname = fn
 208                 for fn in os.listdir(path):
 209                     subname = inspect.getmodulename(fn)
 210                     if subname=='__init__':
 211                         ispkg = True
 212                         break
 213                 else:
 214                     continue    # not a package
 215
 216             if modname and '.' not in modname:
 217                 yielded[modname] = 1
 218                 yield prefix + modname, ispkg
 219
 220
 221 class ImpLoader:
 222     """PEP 302 Loader that wraps Python's "classic" import algorithm
 223     """
 224     code = source = None
 225
 226     def __init__(self, fullname, file, filename, etc):
 227         self.file = file
 228         self.filename = filename
 229         self.fullname = fullname
 230         self.etc = etc
 231
 232     def load_module(self, fullname):
 233         self._reopen()
 234         try:
 235             mod = imp.load_module(fullname, self.file, self.filename, self.etc)
 236         finally:
 237             if self.file:
 238                 self.file.close()
 239         # Note: we don't set __loader__ because we want the module to look
 240         # normal; i.e. this is just a wrapper for standard import machinery
 241         return mod
 242
 243     def get_data(self, pathname):
 244         return open(pathname, "rb").read()
 245
 246     def _reopen(self):
 247         if self.file and self.file.closed:
 248             mod_type = self.etc[2]
 249             if mod_type==imp.PY_SOURCE:
 250                 self.file = open(self.filename, 'rU')
 251             elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
 252                 self.file = open(self.filename, 'rb')
 253
 254     def _fix_name(self, fullname):
 255         if fullname is None:
 256             fullname = self.fullname
 257         elif fullname != self.fullname:
 258             raise ImportError("Loader for module %s cannot handle "
 259                               "module %s" % (self.fullname, fullname))
 260         return fullname
 261
 262     def is_package(self, fullname):
 263         fullname = self._fix_name(fullname)
 264         return self.etc[2]==imp.PKG_DIRECTORY
 265
 266     def get_code(self, fullname=None):
 267         fullname = self._fix_name(fullname)
 268         if self.code is None:
 269             mod_type = self.etc[2]
 270             if mod_type==imp.PY_SOURCE:
 271                 source = self.get_source(fullname)
 272                 self.code = compile(source, self.filename, 'exec')
 273             elif mod_type==imp.PY_COMPILED:
 274                 self._reopen()
 275                 try:
 276                     self.code = read_code(self.file)
 277                 finally:
 278                     self.file.close()
 279             elif mod_type==imp.PKG_DIRECTORY:
 280                 self.code = self._get_delegate().get_code()
 281         return self.code
 282
 283     def get_source(self, fullname=None):
 284         fullname = self._fix_name(fullname)
 285         if self.source is None:
 286             mod_type = self.etc[2]
 287             if mod_type==imp.PY_SOURCE:
 288                 self._reopen()
 289                 try:
 290                     self.source = self.file.read()
 291                 finally:
 292                     self.file.close()
 293             elif mod_type==imp.PY_COMPILED:
 294                 if os.path.exists(self.filename[:-1]):
 295                     f = open(self.filename[:-1], 'rU')
 296                     self.source = f.read()
 297                     f.close()
 298             elif mod_type==imp.PKG_DIRECTORY:
 299                 self.source = self._get_delegate().get_source()
 300         return self.source
 301
 302
 303     def _get_delegate(self):
 304         return ImpImporter(self.filename).find_module('__init__')
 305
 306     def get_filename(self, fullname=None):
 307         fullname = self._fix_name(fullname)
 308         mod_type = self.etc[2]
 309         if self.etc[2]==imp.PKG_DIRECTORY:
 310             return self._get_delegate().get_filename()
 311         elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
 312             return self.filename
 313         return None
 314
 315
 316 try:
 317     import zipimport
 318     from zipimport import zipimporter
 319
 320     def iter_zipimport_modules(importer, prefix=''):
 321         dirlist = zipimport._zip_directory_cache[importer.archive].keys()
 322         dirlist.sort()
 323         _prefix = importer.prefix
 324         plen = len(_prefix)
 325         yielded = {}
 326         import inspect
 327         for fn in dirlist:
 328             if not fn.startswith(_prefix):
 329                 continue
 330
 331             fn = fn[plen:].split(os.sep)
 332
 333             if len(fn)==2 and fn[1].startswith('__init__.py'):
 334                 if fn[0] not in yielded:
 335                     yielded[fn[0]] = 1
 336                     yield fn[0], True
 337
 338             if len(fn)!=1:
 339                 continue
 340
 341             modname = inspect.getmodulename(fn[0])
 342             if modname=='__init__':
 343                 continue
 344
 345             if modname and '.' not in modname and modname not in yielded:
 346                 yielded[modname] = 1
 347                 yield prefix + modname, False
 348
 349     iter_importer_modules.register(zipimporter, iter_zipimport_modules)
 350
 351 except ImportError:
 352     pass
 353
 354
 355 def get_importer(path_item):
 356     """Retrieve a PEP 302 importer for the given path item
 357
 358     The returned importer is cached in sys.path_importer_cache
 359     if it was newly created by a path hook.
 360
 361     If there is no importer, a wrapper around the basic import
 362     machinery is returned. This wrapper is never inserted into
 363     the importer cache (None is inserted instead).
 364
 365     The cache (or part of it) can be cleared manually if a
 366     rescan of sys.path_hooks is necessary.
 367     """
 368     try:
 369         importer = sys.path_importer_cache[path_item]
 370     except KeyError:
 371         for path_hook in sys.path_hooks:
 372             try:
 373                 importer = path_hook(path_item)
 374                 break
 375             except ImportError:
 376                 pass
 377         else:
 378             importer = None
 379         sys.path_importer_cache.setdefault(path_item, importer)
 380
 381     if importer is None:
 382         try:
 383             importer = ImpImporter(path_item)
 384         except ImportError:
 385             importer = None
 386     return importer
 387
 388
 389 def iter_importers(fullname=""):
 390     """Yield PEP 302 importers for the given module name
 391
 392     If fullname contains a '.', the importers will be for the package
 393     containing fullname, otherwise they will be importers for sys.meta_path,
 394     sys.path, and Python's "classic" import machinery, in that order.  If
 395     the named module is in a package, that package is imported as a side
 396     effect of invoking this function.
 397
 398     Non PEP 302 mechanisms (e.g. the Windows registry) used by the
 399     standard import machinery to find files in alternative locations
 400     are partially supported, but are searched AFTER sys.path. Normally,
 401     these locations are searched BEFORE sys.path, preventing sys.path
 402     entries from shadowing them.
 403
 404     For this to cause a visible difference in behaviour, there must
 405     be a module or package name that is accessible via both sys.path
 406     and one of the non PEP 302 file system mechanisms. In this case,
 407     the emulation will find the former version, while the builtin
 408     import mechanism will find the latter.
 409
 410     Items of the following types can be affected by this discrepancy:
 411         imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
 412     """
 413     if fullname.startswith('.'):
 414         raise ImportError("Relative module names not supported")
 415     if '.' in fullname:
 416         # Get the containing package's __path__
 417         pkg = '.'.join(fullname.split('.')[:-1])
 418         if pkg not in sys.modules:
 419             __import__(pkg)
 420         path = getattr(sys.modules[pkg], '__path__', None) or []
 421     else:
 422         for importer in sys.meta_path:
 423             yield importer
 424         path = sys.path
 425     for item in path:
 426         yield get_importer(item)
 427     if '.' not in fullname:
 428         yield ImpImporter()
 429
 430 def get_loader(module_or_name):
 431     """Get a PEP 302 "loader" object for module_or_name
 432
 433     If the module or package is accessible via the normal import
 434     mechanism, a wrapper around the relevant part of that machinery
 435     is returned.  Returns None if the module cannot be found or imported.
 436     If the named module is not already imported, its containing package
 437     (if any) is imported, in order to establish the package __path__.
 438
 439     This function uses iter_importers(), and is thus subject to the same
 440     limitations regarding platform-specific special import locations such
 441     as the Windows registry.
 442     """
 443     if module_or_name in sys.modules:
 444         module_or_name = sys.modules[module_or_name]
 445     if isinstance(module_or_name, ModuleType):
 446         module = module_or_name
 447         loader = getattr(module, '__loader__', None)
 448         if loader is not None:
 449             return loader
 450         fullname = module.__name__
 451     else:
 452         fullname = module_or_name
 453     return find_loader(fullname)
 454
 455 def find_loader(fullname):
 456     """Find a PEP 302 "loader" object for fullname
 457
 458     If fullname contains dots, path must be the containing package's __path__.
 459     Returns None if the module cannot be found or imported. This function uses
 460     iter_importers(), and is thus subject to the same limitations regarding
 461     platform-specific special import locations such as the Windows registry.
 462     """
 463     for importer in iter_importers(fullname):
 464         loader = importer.find_module(fullname)
 465         if loader is not None:
 466             return loader
 467
 468     return None
 469
 470
 471 def extend_path(path, name):
 472     """Extend a package's path.
 473
 474     Intended use is to place the following code in a package's __init__.py:
 475
 476         from pkgutil import extend_path
 477         __path__ = extend_path(__path__, __name__)
 478
 479     This will add to the package's __path__ all subdirectories of
 480     directories on sys.path named after the package.  This is useful
 481     if one wants to distribute different parts of a single logical
 482     package as multiple directories.
 483
 484     It also looks for *.pkg files beginning where * matches the name
 485     argument.  This feature is similar to *.pth files (see site.py),
 486     except that it doesn't special-case lines starting with 'import'.
 487     A *.pkg file is trusted at face value: apart from checking for
 488     duplicates, all entries found in a *.pkg file are added to the
 489     path, regardless of whether they are exist the filesystem.  (This
 490     is a feature.)
 491
 492     If the input path is not a list (as is the case for frozen
 493     packages) it is returned unchanged.  The input path is not
 494     modified; an extended copy is returned.  Items are only appended
 495     to the copy at the end.
 496
 497     It is assumed that sys.path is a sequence.  Items of sys.path that
 498     are not (unicode or 8-bit) strings referring to existing
 499     directories are ignored.  Unicode items of sys.path that cause
 500     errors when used as filenames may cause this function to raise an
 501     exception (in line with os.path.isdir() behavior).
 502     """
 503
 504     if not isinstance(path, list):
 505         # This could happen e.g. when this is called from inside a
 506         # frozen package.  Return the path unchanged in that case.
 507         return path
 508
 509     pname = os.path.join(*name.split('.')) # Reconstitute as relative path
 510     sname_pkg = name + ".pkg"
 511     init_py = "__init__.py"
 512
 513     path = path[:] # Start with a copy of the existing path
 514
 515     for dir in sys.path:
 516         if not isinstance(dir, str) or not os.path.isdir(dir):
 517             continue
 518         subdir = os.path.join(dir, pname)
 519         # XXX This may still add duplicate entries to path on
 520         # case-insensitive filesystems
 521         initfile = os.path.join(subdir, init_py)
 522         if subdir not in path and os.path.isfile(initfile):
 523             path.append(subdir)
 524         # XXX Is this the right thing for subpackages like zope.app?
 525         # It looks for a file named "zope.app.pkg"
 526         pkgfile = os.path.join(dir, sname_pkg)
 527         if os.path.isfile(pkgfile):
 528             try:
 529                 f = open(pkgfile)
 530             except IOError as msg:
 531                 sys.stderr.write("Can't open %s: %s\n" %
 532                                  (pkgfile, msg))
 533             else:
 534                 for line in f:
 535                     line = line.rstrip('\n')
 536                     if not line or line.startswith('#'):
 537                         continue
 538                     path.append(line) # Don't check for existence!
 539                 f.close()
 540
 541     return path
 542
 543 def get_data(package, resource):
 544     """Get a resource from a package.
 545
 546     This is a wrapper round the PEP 302 loader get_data API. The package
 547     argument should be the name of a package, in standard module format
 548     (foo.bar). The resource argument should be in the form of a relative
 549     filename, using '/' as the path separator. The parent directory name '..'
 550     is not allowed, and nor is a rooted name (starting with a '/').
 551
 552     The function returns a binary string, which is the contents of the
 553     specified resource.
 554
 555     For packages located in the filesystem, which have already been imported,
 556     this is the rough equivalent of
 557
 558         d = os.path.dirname(sys.modules[package].__file__)
 559         data = open(os.path.join(d, resource), 'rb').read()
 560
 561     If the package cannot be located or loaded, or it uses a PEP 302 loader
 562     which does not support get_data(), then None is returned.
 563     """
 564
 565     loader = get_loader(package)
 566     if loader is None or not hasattr(loader, 'get_data'):
 567         return None
 568     mod = sys.modules.get(package) or loader.load_module(package)
 569     if mod is None or not hasattr(mod, '__file__'):
 570         return None
 571
 572     # Modify the resource name to be compatible with the loader.get_data
 573     # signature - an os.path format "filename" starting with the dirname of
 574     # the package's __file__
 575     parts = resource.split('/')
 576     parts.insert(0, os.path.dirname(mod.__file__))
 577     resource_name = os.path.join(*parts)
 578     return loader.get_data(resource_name)