Lib/ihooks.py

   1 """Import hook support.
   2
   3 Consistent use of this module will make it possible to change the
   4 different mechanisms involved in loading modules independently.
   5
   6 While the built-in module imp exports interfaces to the built-in
   7 module searching and loading algorithm, and it is possible to replace
   8 the built-in function __import__ in order to change the semantics of
   9 the import statement, until now it has been difficult to combine the
  10 effect of different __import__ hacks, like loading modules from URLs
  11 by rimport.py, or restricted execution by rexec.py.
  12
  13 This module defines three new concepts:
  14
  15 1) A "file system hooks" class provides an interface to a filesystem.
  16
  17 One hooks class is defined (Hooks), which uses the interface provided
  18 by standard modules os and os.path.  It should be used as the base
  19 class for other hooks classes.
  20
  21 2) A "module loader" class provides an interface to search for a
  22 module in a search path and to load it.  It defines a method which
  23 searches for a module in a single directory; by overriding this method
  24 one can redefine the details of the search.  If the directory is None,
  25 built-in and frozen modules are searched instead.
  26
  27 Two module loader class are defined, both implementing the search
  28 strategy used by the built-in __import__ function: ModuleLoader uses
  29 the imp module's find_module interface, while HookableModuleLoader
  30 uses a file system hooks class to interact with the file system.  Both
  31 use the imp module's load_* interfaces to actually load the module.
  32
  33 3) A "module importer" class provides an interface to import a
  34 module, as well as interfaces to reload and unload a module.  It also
  35 provides interfaces to install and uninstall itself instead of the
  36 default __import__ and reload (and unload) functions.
  37
  38 One module importer class is defined (ModuleImporter), which uses a
  39 module loader instance passed in (by default HookableModuleLoader is
  40 instantiated).
  41
  42 The classes defined here should be used as base classes for extended
  43 functionality along those lines.
  44
  45 If a module importer class supports dotted names, its import_module()
  46 must return a different value depending on whether it is called on
  47 behalf of a "from ... import ..." statement or not.  (This is caused
  48 by the way the __import__ hook is used by the Python interpreter.)  It
  49 would also do wise to install a different version of reload().
  50
  51 """
  52 from warnings import warnpy3k
  53 warnpy3k("the ihooks module has been removed in Python 3.0", stacklevel=2)
  54 del warnpy3k
  55
  56 import __builtin__
  57 import imp
  58 import os
  59 import sys
  60
  61 __all__ = ["BasicModuleLoader","Hooks","ModuleLoader","FancyModuleLoader",
  62            "BasicModuleImporter","ModuleImporter","install","uninstall"]
  63
  64 VERBOSE = 0
  65
  66
  67 from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED
  68 from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY
  69 BUILTIN_MODULE = C_BUILTIN
  70 FROZEN_MODULE = PY_FROZEN
  71
  72
  73 class _Verbose:
  74
  75     def __init__(self, verbose = VERBOSE):
  76         self.verbose = verbose
  77
  78     def get_verbose(self):
  79         return self.verbose
  80
  81     def set_verbose(self, verbose):
  82         self.verbose = verbose
  83
  84     # XXX The following is an experimental interface
  85
  86     def note(self, *args):
  87         if self.verbose:
  88             self.message(*args)
  89
  90     def message(self, format, *args):
  91         if args:
  92             print format%args
  93         else:
  94             print format
  95
  96
  97 class BasicModuleLoader(_Verbose):
  98
  99     """Basic module loader.
 100
 101     This provides the same functionality as built-in import.  It
 102     doesn't deal with checking sys.modules -- all it provides is
 103     find_module() and a load_module(), as well as find_module_in_dir()
 104     which searches just one directory, and can be overridden by a
 105     derived class to change the module search algorithm when the basic
 106     dependency on sys.path is unchanged.
 107
 108     The interface is a little more convenient than imp's:
 109     find_module(name, [path]) returns None or 'stuff', and
 110     load_module(name, stuff) loads the module.
 111
 112     """
 113
 114     def find_module(self, name, path = None):
 115         if path is None:
 116             path = [None] + self.default_path()
 117         for dir in path:
 118             stuff = self.find_module_in_dir(name, dir)
 119             if stuff: return stuff
 120         return None
 121
 122     def default_path(self):
 123         return sys.path
 124
 125     def find_module_in_dir(self, name, dir):
 126         if dir is None:
 127             return self.find_builtin_module(name)
 128         else:
 129             try:
 130                 return imp.find_module(name, [dir])
 131             except ImportError:
 132                 return None
 133
 134     def find_builtin_module(self, name):
 135         # XXX frozen packages?
 136         if imp.is_builtin(name):
 137             return None, '', ('', '', BUILTIN_MODULE)
 138         if imp.is_frozen(name):
 139             return None, '', ('', '', FROZEN_MODULE)
 140         return None
 141
 142     def load_module(self, name, stuff):
 143         file, filename, info = stuff
 144         try:
 145             return imp.load_module(name, file, filename, info)
 146         finally:
 147             if file: file.close()
 148
 149
 150 class Hooks(_Verbose):
 151
 152     """Hooks into the filesystem and interpreter.
 153
 154     By deriving a subclass you can redefine your filesystem interface,
 155     e.g. to merge it with the URL space.
 156
 157     This base class behaves just like the native filesystem.
 158
 159     """
 160
 161     # imp interface
 162     def get_suffixes(self): return imp.get_suffixes()
 163     def new_module(self, name): return imp.new_module(name)
 164     def is_builtin(self, name): return imp.is_builtin(name)
 165     def init_builtin(self, name): return imp.init_builtin(name)
 166     def is_frozen(self, name): return imp.is_frozen(name)
 167     def init_frozen(self, name): return imp.init_frozen(name)
 168     def get_frozen_object(self, name): return imp.get_frozen_object(name)
 169     def load_source(self, name, filename, file=None):
 170         return imp.load_source(name, filename, file)
 171     def load_compiled(self, name, filename, file=None):
 172         return imp.load_compiled(name, filename, file)
 173     def load_dynamic(self, name, filename, file=None):
 174         return imp.load_dynamic(name, filename, file)
 175     def load_package(self, name, filename, file=None):
 176         return imp.load_module(name, file, filename, ("", "", PKG_DIRECTORY))
 177
 178     def add_module(self, name):
 179         d = self.modules_dict()
 180         if name in d: return d[name]
 181         d[name] = m = self.new_module(name)
 182         return m
 183
 184     # sys interface
 185     def modules_dict(self): return sys.modules
 186     def default_path(self): return sys.path
 187
 188     def path_split(self, x): return os.path.split(x)
 189     def path_join(self, x, y): return os.path.join(x, y)
 190     def path_isabs(self, x): return os.path.isabs(x)
 191     # etc.
 192
 193     def path_exists(self, x): return os.path.exists(x)
 194     def path_isdir(self, x): return os.path.isdir(x)
 195     def path_isfile(self, x): return os.path.isfile(x)
 196     def path_islink(self, x): return os.path.islink(x)
 197     # etc.
 198
 199     def openfile(self, *x): return open(*x)
 200     openfile_error = IOError
 201     def listdir(self, x): return os.listdir(x)
 202     listdir_error = os.error
 203     # etc.
 204
 205
 206 class ModuleLoader(BasicModuleLoader):
 207
 208     """Default module loader; uses file system hooks.
 209
 210     By defining suitable hooks, you might be able to load modules from
 211     other sources than the file system, e.g. from compressed or
 212     encrypted files, tar files or (if you're brave!) URLs.
 213
 214     """
 215
 216     def __init__(self, hooks = None, verbose = VERBOSE):
 217         BasicModuleLoader.__init__(self, verbose)
 218         self.hooks = hooks or Hooks(verbose)
 219
 220     def default_path(self):
 221         return self.hooks.default_path()
 222
 223     def modules_dict(self):
 224         return self.hooks.modules_dict()
 225
 226     def get_hooks(self):
 227         return self.hooks
 228
 229     def set_hooks(self, hooks):
 230         self.hooks = hooks
 231
 232     def find_builtin_module(self, name):
 233         # XXX frozen packages?
 234         if self.hooks.is_builtin(name):
 235             return None, '', ('', '', BUILTIN_MODULE)
 236         if self.hooks.is_frozen(name):
 237             return None, '', ('', '', FROZEN_MODULE)
 238         return None
 239
 240     def find_module_in_dir(self, name, dir, allow_packages=1):
 241         if dir is None:
 242             return self.find_builtin_module(name)
 243         if allow_packages:
 244             fullname = self.hooks.path_join(dir, name)
 245             if self.hooks.path_isdir(fullname):
 246                 stuff = self.find_module_in_dir("__init__", fullname, 0)
 247                 if stuff:
 248                     file = stuff[0]
 249                     if file: file.close()
 250                     return None, fullname, ('', '', PKG_DIRECTORY)
 251         for info in self.hooks.get_suffixes():
 252             suff, mode, type = info
 253             fullname = self.hooks.path_join(dir, name+suff)
 254             try:
 255                 fp = self.hooks.openfile(fullname, mode)
 256                 return fp, fullname, info
 257             except self.hooks.openfile_error:
 258                 pass
 259         return None
 260
 261     def load_module(self, name, stuff):
 262         file, filename, info = stuff
 263         (suff, mode, type) = info
 264         try:
 265             if type == BUILTIN_MODULE:
 266                 return self.hooks.init_builtin(name)
 267             if type == FROZEN_MODULE:
 268                 return self.hooks.init_frozen(name)
 269             if type == C_EXTENSION:
 270                 m = self.hooks.load_dynamic(name, filename, file)
 271             elif type == PY_SOURCE:
 272                 m = self.hooks.load_source(name, filename, file)
 273             elif type == PY_COMPILED:
 274                 m = self.hooks.load_compiled(name, filename, file)
 275             elif type == PKG_DIRECTORY:
 276                 m = self.hooks.load_package(name, filename, file)
 277             else:
 278                 raise ImportError, "Unrecognized module type (%r) for %s" % \
 279                       (type, name)
 280         finally:
 281             if file: file.close()
 282         m.__file__ = filename
 283         return m
 284
 285
 286 class FancyModuleLoader(ModuleLoader):
 287
 288     """Fancy module loader -- parses and execs the code itself."""
 289
 290     def load_module(self, name, stuff):
 291         file, filename, (suff, mode, type) = stuff
 292         realfilename = filename
 293         path = None
 294
 295         if type == PKG_DIRECTORY:
 296             initstuff = self.find_module_in_dir("__init__", filename, 0)
 297             if not initstuff:
 298                 raise ImportError, "No __init__ module in package %s" % name
 299             initfile, initfilename, initinfo = initstuff
 300             initsuff, initmode, inittype = initinfo
 301             if inittype not in (PY_COMPILED, PY_SOURCE):
 302                 if initfile: initfile.close()
 303                 raise ImportError, \
 304                     "Bad type (%r) for __init__ module in package %s" % (
 305                     inittype, name)
 306             path = [filename]
 307             file = initfile
 308             realfilename = initfilename
 309             type = inittype
 310
 311         if type == FROZEN_MODULE:
 312             code = self.hooks.get_frozen_object(name)
 313         elif type == PY_COMPILED:
 314             import marshal
 315             file.seek(8)
 316             code = marshal.load(file)
 317         elif type == PY_SOURCE:
 318             data = file.read()
 319             code = compile(data, realfilename, 'exec')
 320         else:
 321             return ModuleLoader.load_module(self, name, stuff)
 322
 323         m = self.hooks.add_module(name)
 324         if path:
 325             m.__path__ = path
 326         m.__file__ = filename
 327         try:
 328             exec code in m.__dict__
 329         except:
 330             d = self.hooks.modules_dict()
 331             if name in d:
 332                 del d[name]
 333             raise
 334         return m
 335
 336
 337 class BasicModuleImporter(_Verbose):
 338
 339     """Basic module importer; uses module loader.
 340
 341     This provides basic import facilities but no package imports.
 342
 343     """
 344
 345     def __init__(self, loader = None, verbose = VERBOSE):
 346         _Verbose.__init__(self, verbose)
 347         self.loader = loader or ModuleLoader(None, verbose)
 348         self.modules = self.loader.modules_dict()
 349
 350     def get_loader(self):
 351         return self.loader
 352
 353     def set_loader(self, loader):
 354         self.loader = loader
 355
 356     def get_hooks(self):
 357         return self.loader.get_hooks()
 358
 359     def set_hooks(self, hooks):
 360         return self.loader.set_hooks(hooks)
 361
 362     def import_module(self, name, globals={}, locals={}, fromlist=[]):
 363         name = str(name)
 364         if name in self.modules:
 365             return self.modules[name] # Fast path
 366         stuff = self.loader.find_module(name)
 367         if not stuff:
 368             raise ImportError, "No module named %s" % name
 369         return self.loader.load_module(name, stuff)
 370
 371     def reload(self, module, path = None):
 372         name = str(module.__name__)
 373         stuff = self.loader.find_module(name, path)
 374         if not stuff:
 375             raise ImportError, "Module %s not found for reload" % name
 376         return self.loader.load_module(name, stuff)
 377
 378     def unload(self, module):
 379         del self.modules[str(module.__name__)]
 380         # XXX Should this try to clear the module's namespace?
 381
 382     def install(self):
 383         self.save_import_module = __builtin__.__import__
 384         self.save_reload = __builtin__.reload
 385         if not hasattr(__builtin__, 'unload'):
 386             __builtin__.unload = None
 387         self.save_unload = __builtin__.unload
 388         __builtin__.__import__ = self.import_module
 389         __builtin__.reload = self.reload
 390         __builtin__.unload = self.unload
 391
 392     def uninstall(self):
 393         __builtin__.__import__ = self.save_import_module
 394         __builtin__.reload = self.save_reload
 395         __builtin__.unload = self.save_unload
 396         if not __builtin__.unload:
 397             del __builtin__.unload
 398
 399
 400 class ModuleImporter(BasicModuleImporter):
 401
 402     """A module importer that supports packages."""
 403
 404     def import_module(self, name, globals=None, locals=None, fromlist=None):
 405         parent = self.determine_parent(globals)
 406         q, tail = self.find_head_package(parent, str(name))
 407         m = self.load_tail(q, tail)
 408         if not fromlist:
 409             return q
 410         if hasattr(m, "__path__"):
 411             self.ensure_fromlist(m, fromlist)
 412         return m
 413
 414     def determine_parent(self, globals):
 415         if not globals or not "__name__" in globals:
 416             return None
 417         pname = globals['__name__']
 418         if "__path__" in globals:
 419             parent = self.modules[pname]
 420             assert globals is parent.__dict__
 421             return parent
 422         if '.' in pname:
 423             i = pname.rfind('.')
 424             pname = pname[:i]
 425             parent = self.modules[pname]
 426             assert parent.__name__ == pname
 427             return parent
 428         return None
 429
 430     def find_head_package(self, parent, name):
 431         if '.' in name:
 432             i = name.find('.')
 433             head = name[:i]
 434             tail = name[i+1:]
 435         else:
 436             head = name
 437             tail = ""
 438         if parent:
 439             qname = "%s.%s" % (parent.__name__, head)
 440         else:
 441             qname = head
 442         q = self.import_it(head, qname, parent)
 443         if q: return q, tail
 444         if parent:
 445             qname = head
 446             parent = None
 447             q = self.import_it(head, qname, parent)
 448             if q: return q, tail
 449         raise ImportError, "No module named " + qname
 450
 451     def load_tail(self, q, tail):
 452         m = q
 453         while tail:
 454             i = tail.find('.')
 455             if i < 0: i = len(tail)
 456             head, tail = tail[:i], tail[i+1:]
 457             mname = "%s.%s" % (m.__name__, head)
 458             m = self.import_it(head, mname, m)
 459             if not m:
 460                 raise ImportError, "No module named " + mname
 461         return m
 462
 463     def ensure_fromlist(self, m, fromlist, recursive=0):
 464         for sub in fromlist:
 465             if sub == "*":
 466                 if not recursive:
 467                     try:
 468                         all = m.__all__
 469                     except AttributeError:
 470                         pass
 471                     else:
 472                         self.ensure_fromlist(m, all, 1)
 473                 continue
 474             if sub != "*" and not hasattr(m, sub):
 475                 subname = "%s.%s" % (m.__name__, sub)
 476                 submod = self.import_it(sub, subname, m)
 477                 if not submod:
 478                     raise ImportError, "No module named " + subname
 479
 480     def import_it(self, partname, fqname, parent, force_load=0):
 481         if not partname:
 482             raise ValueError, "Empty module name"
 483         if not force_load:
 484             try:
 485                 return self.modules[fqname]
 486             except KeyError:
 487                 pass
 488         try:
 489             path = parent and parent.__path__
 490         except AttributeError:
 491             return None
 492         partname = str(partname)
 493         stuff = self.loader.find_module(partname, path)
 494         if not stuff:
 495             return None
 496         fqname = str(fqname)
 497         m = self.loader.load_module(fqname, stuff)
 498         if parent:
 499             setattr(parent, partname, m)
 500         return m
 501
 502     def reload(self, module):
 503         name = str(module.__name__)
 504         if '.' not in name:
 505             return self.import_it(name, name, None, force_load=1)
 506         i = name.rfind('.')
 507         pname = name[:i]
 508         parent = self.modules[pname]
 509         return self.import_it(name[i+1:], name, parent, force_load=1)
 510
 511
 512 default_importer = None
 513 current_importer = None
 514
 515 def install(importer = None):
 516     global current_importer
 517     current_importer = importer or default_importer or ModuleImporter()
 518     current_importer.install()
 519
 520 def uninstall():
 521     global current_importer
 522     current_importer.uninstall()