Lib/ihooks.py

   1 """Import hook support.
   2
   3 Consistent use of this module will make it possible to change the
   4 different mechanisms involved in loading modules independently.
   5
   6 While the built-in module imp exports interfaces to the built-in
   7 module searching and loading algorithm, and it is possible to replace
   8 the built-in function __import__ in order to change the semantics of
   9 the import statement, until now it has been difficult to combine the
  10 effect of different __import__ hacks, like loading modules from URLs
  11 by rimport.py, or restricted execution by rexec.py.
  12
  13 This module defines three new concepts:
  14
  15 1) A "file system hooks" class provides an interface to a filesystem.
  16
  17 One hooks class is defined (Hooks), which uses the interface provided
  18 by standard modules os and os.path.  It should be used as the base
  19 class for other hooks classes.
  20
  21 2) A "module loader" class provides an interface to search for a
  22 module in a search path and to load it.  It defines a method which
  23 searches for a module in a single directory; by overriding this method
  24 one can redefine the details of the search.  If the directory is None,
  25 built-in and frozen modules are searched instead.
  26
  27 Two module loader class are defined, both implementing the search
  28 strategy used by the built-in __import__ function: ModuleLoader uses
  29 the imp module's find_module interface, while HookableModuleLoader
  30 uses a file system hooks class to interact with the file system.  Both
  31 use the imp module's load_* interfaces to actually load the module.
  32
  33 3) A "module importer" class provides an interface to import a
  34 module, as well as interfaces to reload and unload a module.  It also
  35 provides interfaces to install and uninstall itself instead of the
  36 default __import__ and reload (and unload) functions.
  37
  38 One module importer class is defined (ModuleImporter), which uses a
  39 module loader instance passed in (by default HookableModuleLoader is
  40 instantiated).
  41
  42 The classes defined here should be used as base classes for extended
  43 functionality along those lines.
  44
  45 If a module importer class supports dotted names, its import_module()
  46 must return a different value depending on whether it is called on
  47 behalf of a "from ... import ..." statement or not.  (This is caused
  48 by the way the __import__ hook is used by the Python interpreter.)  It
  49 would also do wise to install a different version of reload().
  50
  51 """
  52
  53
  54 import __builtin__
  55 import imp
  56 import os
  57 import sys
  58
  59 __all__ = ["BasicModuleLoader","Hooks","ModuleLoader","FancyModuleLoader",
  60            "BasicModuleImporter","ModuleImporter","install","uninstall"]
  61
  62 VERBOSE = 0
  63
  64
  65 from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED
  66 from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY
  67 BUILTIN_MODULE = C_BUILTIN
  68 FROZEN_MODULE = PY_FROZEN
  69
  70
  71 class _Verbose:
  72
  73     def __init__(self, verbose = VERBOSE):
  74         self.verbose = verbose
  75
  76     def get_verbose(self):
  77         return self.verbose
  78
  79     def set_verbose(self, verbose):
  80         self.verbose = verbose
  81
  82     # XXX The following is an experimental interface
  83
  84     def note(self, *args):
  85         if self.verbose:
  86             self.message(*args)
  87
  88     def message(self, format, *args):
  89         if args:
  90             print format%args
  91         else:
  92             print format
  93
  94
  95 class BasicModuleLoader(_Verbose):
  96
  97     """Basic module loader.
  98
  99     This provides the same functionality as built-in import.  It
 100     doesn't deal with checking sys.modules -- all it provides is
 101     find_module() and a load_module(), as well as find_module_in_dir()
 102     which searches just one directory, and can be overridden by a
 103     derived class to change the module search algorithm when the basic
 104     dependency on sys.path is unchanged.
 105
 106     The interface is a little more convenient than imp's:
 107     find_module(name, [path]) returns None or 'stuff', and
 108     load_module(name, stuff) loads the module.
 109
 110     """
 111
 112     def find_module(self, name, path = None):
 113         if path is None:
 114             path = [None] + self.default_path()
 115         for dir in path:
 116             stuff = self.find_module_in_dir(name, dir)
 117             if stuff: return stuff
 118         return None
 119
 120     def default_path(self):
 121         return sys.path
 122
 123     def find_module_in_dir(self, name, dir):
 124         if dir is None:
 125             return self.find_builtin_module(name)
 126         else:
 127             try:
 128                 return imp.find_module(name, [dir])
 129             except ImportError:
 130                 return None
 131
 132     def find_builtin_module(self, name):
 133         # XXX frozen packages?
 134         if imp.is_builtin(name):
 135             return None, '', ('', '', BUILTIN_MODULE)
 136         if imp.is_frozen(name):
 137             return None, '', ('', '', FROZEN_MODULE)
 138         return None
 139
 140     def load_module(self, name, stuff):
 141         file, filename, info = stuff
 142         try:
 143             return imp.load_module(name, file, filename, info)
 144         finally:
 145             if file: file.close()
 146
 147
 148 class Hooks(_Verbose):
 149
 150     """Hooks into the filesystem and interpreter.
 151
 152     By deriving a subclass you can redefine your filesystem interface,
 153     e.g. to merge it with the URL space.
 154
 155     This base class behaves just like the native filesystem.
 156
 157     """
 158
 159     # imp interface
 160     def get_suffixes(self): return imp.get_suffixes()
 161     def new_module(self, name): return imp.new_module(name)
 162     def is_builtin(self, name): return imp.is_builtin(name)
 163     def init_builtin(self, name): return imp.init_builtin(name)
 164     def is_frozen(self, name): return imp.is_frozen(name)
 165     def init_frozen(self, name): return imp.init_frozen(name)
 166     def get_frozen_object(self, name): return imp.get_frozen_object(name)
 167     def load_source(self, name, filename, file=None):
 168         return imp.load_source(name, filename, file)
 169     def load_compiled(self, name, filename, file=None):
 170         return imp.load_compiled(name, filename, file)
 171     def load_dynamic(self, name, filename, file=None):
 172         return imp.load_dynamic(name, filename, file)
 173     def load_package(self, name, filename, file=None):
 174         return imp.load_module(name, file, filename, ("", "", PKG_DIRECTORY))
 175
 176     def add_module(self, name):
 177         d = self.modules_dict()
 178         if name in d: return d[name]
 179         d[name] = m = self.new_module(name)
 180         return m
 181
 182     # sys interface
 183     def modules_dict(self): return sys.modules
 184     def default_path(self): return sys.path
 185
 186     def path_split(self, x): return os.path.split(x)
 187     def path_join(self, x, y): return os.path.join(x, y)
 188     def path_isabs(self, x): return os.path.isabs(x)
 189     # etc.
 190
 191     def path_exists(self, x): return os.path.exists(x)
 192     def path_isdir(self, x): return os.path.isdir(x)
 193     def path_isfile(self, x): return os.path.isfile(x)
 194     def path_islink(self, x): return os.path.islink(x)
 195     # etc.
 196
 197     def openfile(self, *x): return open(*x)
 198     openfile_error = IOError
 199     def listdir(self, x): return os.listdir(x)
 200     listdir_error = os.error
 201     # etc.
 202
 203
 204 class ModuleLoader(BasicModuleLoader):
 205
 206     """Default module loader; uses file system hooks.
 207
 208     By defining suitable hooks, you might be able to load modules from
 209     other sources than the file system, e.g. from compressed or
 210     encrypted files, tar files or (if you're brave!) URLs.
 211
 212     """
 213
 214     def __init__(self, hooks = None, verbose = VERBOSE):
 215         BasicModuleLoader.__init__(self, verbose)
 216         self.hooks = hooks or Hooks(verbose)
 217
 218     def default_path(self):
 219         return self.hooks.default_path()
 220
 221     def modules_dict(self):
 222         return self.hooks.modules_dict()
 223
 224     def get_hooks(self):
 225         return self.hooks
 226
 227     def set_hooks(self, hooks):
 228         self.hooks = hooks
 229
 230     def find_builtin_module(self, name):
 231         # XXX frozen packages?
 232         if self.hooks.is_builtin(name):
 233             return None, '', ('', '', BUILTIN_MODULE)
 234         if self.hooks.is_frozen(name):
 235             return None, '', ('', '', FROZEN_MODULE)
 236         return None
 237
 238     def find_module_in_dir(self, name, dir, allow_packages=1):
 239         if dir is None:
 240             return self.find_builtin_module(name)
 241         if allow_packages:
 242             fullname = self.hooks.path_join(dir, name)
 243             if self.hooks.path_isdir(fullname):
 244                 stuff = self.find_module_in_dir("__init__", fullname, 0)
 245                 if stuff:
 246                     file = stuff[0]
 247                     if file: file.close()
 248                     return None, fullname, ('', '', PKG_DIRECTORY)
 249         for info in self.hooks.get_suffixes():
 250             suff, mode, type = info
 251             fullname = self.hooks.path_join(dir, name+suff)
 252             try:
 253                 fp = self.hooks.openfile(fullname, mode)
 254                 return fp, fullname, info
 255             except self.hooks.openfile_error:
 256                 pass
 257         return None
 258
 259     def load_module(self, name, stuff):
 260         file, filename, info = stuff
 261         (suff, mode, type) = info
 262         try:
 263             if type == BUILTIN_MODULE:
 264                 return self.hooks.init_builtin(name)
 265             if type == FROZEN_MODULE:
 266                 return self.hooks.init_frozen(name)
 267             if type == C_EXTENSION:
 268                 m = self.hooks.load_dynamic(name, filename, file)
 269             elif type == PY_SOURCE:
 270                 m = self.hooks.load_source(name, filename, file)
 271             elif type == PY_COMPILED:
 272                 m = self.hooks.load_compiled(name, filename, file)
 273             elif type == PKG_DIRECTORY:
 274                 m = self.hooks.load_package(name, filename, file)
 275             else:
 276                 raise ImportError, "Unrecognized module type (%r) for %s" % \
 277                       (type, name)
 278         finally:
 279             if file: file.close()
 280         m.__file__ = filename
 281         return m
 282
 283
 284 class FancyModuleLoader(ModuleLoader):
 285
 286     """Fancy module loader -- parses and execs the code itself."""
 287
 288     def load_module(self, name, stuff):
 289         file, filename, (suff, mode, type) = stuff
 290         realfilename = filename
 291         path = None
 292
 293         if type == PKG_DIRECTORY:
 294             initstuff = self.find_module_in_dir("__init__", filename, 0)
 295             if not initstuff:
 296                 raise ImportError, "No __init__ module in package %s" % name
 297             initfile, initfilename, initinfo = initstuff
 298             initsuff, initmode, inittype = initinfo
 299             if inittype not in (PY_COMPILED, PY_SOURCE):
 300                 if initfile: initfile.close()
 301                 raise ImportError, \
 302                     "Bad type (%r) for __init__ module in package %s" % (
 303                     inittype, name)
 304             path = [filename]
 305             file = initfile
 306             realfilename = initfilename
 307             type = inittype
 308
 309         if type == FROZEN_MODULE:
 310             code = self.hooks.get_frozen_object(name)
 311         elif type == PY_COMPILED:
 312             import marshal
 313             file.seek(8)
 314             code = marshal.load(file)
 315         elif type == PY_SOURCE:
 316             data = file.read()
 317             code = compile(data, realfilename, 'exec')
 318         else:
 319             return ModuleLoader.load_module(self, name, stuff)
 320
 321         m = self.hooks.add_module(name)
 322         if path:
 323             m.__path__ = path
 324         m.__file__ = filename
 325         try:
 326             exec code in m.__dict__
 327         except:
 328             d = self.hooks.modules_dict()
 329             if name in d:
 330                 del d[name]
 331             raise
 332         return m
 333
 334
 335 class BasicModuleImporter(_Verbose):
 336
 337     """Basic module importer; uses module loader.
 338
 339     This provides basic import facilities but no package imports.
 340
 341     """
 342
 343     def __init__(self, loader = None, verbose = VERBOSE):
 344         _Verbose.__init__(self, verbose)
 345         self.loader = loader or ModuleLoader(None, verbose)
 346         self.modules = self.loader.modules_dict()
 347
 348     def get_loader(self):
 349         return self.loader
 350
 351     def set_loader(self, loader):
 352         self.loader = loader
 353
 354     def get_hooks(self):
 355         return self.loader.get_hooks()
 356
 357     def set_hooks(self, hooks):
 358         return self.loader.set_hooks(hooks)
 359
 360     def import_module(self, name, globals={}, locals={}, fromlist=[]):
 361         name = str(name)
 362         if name in self.modules:
 363             return self.modules[name] # Fast path
 364         stuff = self.loader.find_module(name)
 365         if not stuff:
 366             raise ImportError, "No module named %s" % name
 367         return self.loader.load_module(name, stuff)
 368
 369     def reload(self, module, path = None):
 370         name = str(module.__name__)
 371         stuff = self.loader.find_module(name, path)
 372         if not stuff:
 373             raise ImportError, "Module %s not found for reload" % name
 374         return self.loader.load_module(name, stuff)
 375
 376     def unload(self, module):
 377         del self.modules[str(module.__name__)]
 378         # XXX Should this try to clear the module's namespace?
 379
 380     def install(self):
 381         self.save_import_module = __builtin__.__import__
 382         self.save_reload = __builtin__.reload
 383         if not hasattr(__builtin__, 'unload'):
 384             __builtin__.unload = None
 385         self.save_unload = __builtin__.unload
 386         __builtin__.__import__ = self.import_module
 387         __builtin__.reload = self.reload
 388         __builtin__.unload = self.unload
 389
 390     def uninstall(self):
 391         __builtin__.__import__ = self.save_import_module
 392         __builtin__.reload = self.save_reload
 393         __builtin__.unload = self.save_unload
 394         if not __builtin__.unload:
 395             del __builtin__.unload
 396
 397
 398 class ModuleImporter(BasicModuleImporter):
 399
 400     """A module importer that supports packages."""
 401
 402     def import_module(self, name, globals=None, locals=None, fromlist=None):
 403         parent = self.determine_parent(globals)
 404         q, tail = self.find_head_package(parent, str(name))
 405         m = self.load_tail(q, tail)
 406         if not fromlist:
 407             return q
 408         if hasattr(m, "__path__"):
 409             self.ensure_fromlist(m, fromlist)
 410         return m
 411
 412     def determine_parent(self, globals):
 413         if not globals or not "__name__" in globals:
 414             return None
 415         pname = globals['__name__']
 416         if "__path__" in globals:
 417             parent = self.modules[pname]
 418             assert globals is parent.__dict__
 419             return parent
 420         if '.' in pname:
 421             i = pname.rfind('.')
 422             pname = pname[:i]
 423             parent = self.modules[pname]
 424             assert parent.__name__ == pname
 425             return parent
 426         return None
 427
 428     def find_head_package(self, parent, name):
 429         if '.' in name:
 430             i = name.find('.')
 431             head = name[:i]
 432             tail = name[i+1:]
 433         else:
 434             head = name
 435             tail = ""
 436         if parent:
 437             qname = "%s.%s" % (parent.__name__, head)
 438         else:
 439             qname = head
 440         q = self.import_it(head, qname, parent)
 441         if q: return q, tail
 442         if parent:
 443             qname = head
 444             parent = None
 445             q = self.import_it(head, qname, parent)
 446             if q: return q, tail
 447         raise ImportError, "No module named " + qname
 448
 449     def load_tail(self, q, tail):
 450         m = q
 451         while tail:
 452             i = tail.find('.')
 453             if i < 0: i = len(tail)
 454             head, tail = tail[:i], tail[i+1:]
 455             mname = "%s.%s" % (m.__name__, head)
 456             m = self.import_it(head, mname, m)
 457             if not m:
 458                 raise ImportError, "No module named " + mname
 459         return m
 460
 461     def ensure_fromlist(self, m, fromlist, recursive=0):
 462         for sub in fromlist:
 463             if sub == "*":
 464                 if not recursive:
 465                     try:
 466                         all = m.__all__
 467                     except AttributeError:
 468                         pass
 469                     else:
 470                         self.ensure_fromlist(m, all, 1)
 471                 continue
 472             if sub != "*" and not hasattr(m, sub):
 473                 subname = "%s.%s" % (m.__name__, sub)
 474                 submod = self.import_it(sub, subname, m)
 475                 if not submod:
 476                     raise ImportError, "No module named " + subname
 477
 478     def import_it(self, partname, fqname, parent, force_load=0):
 479         if not partname:
 480             raise ValueError, "Empty module name"
 481         if not force_load:
 482             try:
 483                 return self.modules[fqname]
 484             except KeyError:
 485                 pass
 486         try:
 487             path = parent and parent.__path__
 488         except AttributeError:
 489             return None
 490         partname = str(partname)
 491         stuff = self.loader.find_module(partname, path)
 492         if not stuff:
 493             return None
 494         fqname = str(fqname)
 495         m = self.loader.load_module(fqname, stuff)
 496         if parent:
 497             setattr(parent, partname, m)
 498         return m
 499
 500     def reload(self, module):
 501         name = str(module.__name__)
 502         if '.' not in name:
 503             return self.import_it(name, name, None, force_load=1)
 504         i = name.rfind('.')
 505         pname = name[:i]
 506         parent = self.modules[pname]
 507         return self.import_it(name[i+1:], name, parent, force_load=1)
 508
 509
 510 default_importer = None
 511 current_importer = None
 512
 513 def install(importer = None):
 514     global current_importer
 515     current_importer = importer or default_importer or ModuleImporter()
 516     current_importer.install()
 517
 518 def uninstall():
 519     global current_importer
 520     current_importer.uninstall()