Lib/ihooks.py

   1 """Import hook support.
   2
   3 Consistent use of this module will make it possible to change the
   4 different mechanisms involved in loading modules independently.
   5
   6 While the built-in module imp exports interfaces to the built-in
   7 module searching and loading algorithm, and it is possible to replace
   8 the built-in function __import__ in order to change the semantics of
   9 the import statement, until now it has been difficult to combine the
  10 effect of different __import__ hacks, like loading modules from URLs
  11 by rimport.py, or restricted execution by rexec.py.
  12
  13 This module defines three new concepts:
  14
  15 1) A "file system hooks" class provides an interface to a filesystem.
  16
  17 One hooks class is defined (Hooks), which uses the interface provided
  18 by standard modules os and os.path.  It should be used as the base
  19 class for other hooks classes.
  20
  21 2) A "module loader" class provides an interface to search for a
  22 module in a search path and to load it.  It defines a method which
  23 searches for a module in a single directory; by overriding this method
  24 one can redefine the details of the search.  If the directory is None,
  25 built-in and frozen modules are searched instead.
  26
  27 Two module loader class are defined, both implementing the search
  28 strategy used by the built-in __import__ function: ModuleLoader uses
  29 the imp module's find_module interface, while HookableModuleLoader
  30 uses a file system hooks class to interact with the file system.  Both
  31 use the imp module's load_* interfaces to actually load the module.
  32
  33 3) A "module importer" class provides an interface to import a
  34 module, as well as interfaces to reload and unload a module.  It also
  35 provides interfaces to install and uninstall itself instead of the
  36 default __import__ and reload (and unload) functions.
  37
  38 One module importer class is defined (ModuleImporter), which uses a
  39 module loader instance passed in (by default HookableModuleLoader is
  40 instantiated).
  41
  42 The classes defined here should be used as base classes for extended
  43 functionality along those lines.
  44
  45 If a module importer class supports dotted names, its import_module()
  46 must return a different value depending on whether it is called on
  47 behalf of a "from ... import ..." statement or not.  (This is caused
  48 by the way the __import__ hook is used by the Python interpreter.)  It
  49 would also do wise to install a different version of reload().
  50
  51 """
  52 from warnings import warnpy3k, warn
  53 warnpy3k("the ihooks module has been removed in Python 3.0", stacklevel=2)
  54 del warnpy3k
  55
  56 import __builtin__
  57 import imp
  58 import os
  59 import sys
  60
  61 __all__ = ["BasicModuleLoader","Hooks","ModuleLoader","FancyModuleLoader",
  62            "BasicModuleImporter","ModuleImporter","install","uninstall"]
  63
  64 VERBOSE = 0
  65
  66
  67 from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED
  68 from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY
  69 BUILTIN_MODULE = C_BUILTIN
  70 FROZEN_MODULE = PY_FROZEN
  71
  72
  73 class _Verbose:
  74
  75     def __init__(self, verbose = VERBOSE):
  76         self.verbose = verbose
  77
  78     def get_verbose(self):
  79         return self.verbose
  80
  81     def set_verbose(self, verbose):
  82         self.verbose = verbose
  83
  84     # XXX The following is an experimental interface
  85
  86     def note(self, *args):
  87         if self.verbose:
  88             self.message(*args)
  89
  90     def message(self, format, *args):
  91         if args:
  92             print format%args
  93         else:
  94             print format
  95
  96
  97 class BasicModuleLoader(_Verbose):
  98
  99     """Basic module loader.
 100
 101     This provides the same functionality as built-in import.  It
 102     doesn't deal with checking sys.modules -- all it provides is
 103     find_module() and a load_module(), as well as find_module_in_dir()
 104     which searches just one directory, and can be overridden by a
 105     derived class to change the module search algorithm when the basic
 106     dependency on sys.path is unchanged.
 107
 108     The interface is a little more convenient than imp's:
 109     find_module(name, [path]) returns None or 'stuff', and
 110     load_module(name, stuff) loads the module.
 111
 112     """
 113
 114     def find_module(self, name, path = None):
 115         if path is None:
 116             path = [None] + self.default_path()
 117         for dir in path:
 118             stuff = self.find_module_in_dir(name, dir)
 119             if stuff: return stuff
 120         return None
 121
 122     def default_path(self):
 123         return sys.path
 124
 125     def find_module_in_dir(self, name, dir):
 126         if dir is None:
 127             return self.find_builtin_module(name)
 128         else:
 129             try:
 130                 return imp.find_module(name, [dir])
 131             except ImportError:
 132                 return None
 133
 134     def find_builtin_module(self, name):
 135         # XXX frozen packages?
 136         if imp.is_builtin(name):
 137             return None, '', ('', '', BUILTIN_MODULE)
 138         if imp.is_frozen(name):
 139             return None, '', ('', '', FROZEN_MODULE)
 140         return None
 141
 142     def load_module(self, name, stuff):
 143         file, filename, info = stuff
 144         try:
 145             return imp.load_module(name, file, filename, info)
 146         finally:
 147             if file: file.close()
 148
 149
 150 class Hooks(_Verbose):
 151
 152     """Hooks into the filesystem and interpreter.
 153
 154     By deriving a subclass you can redefine your filesystem interface,
 155     e.g. to merge it with the URL space.
 156
 157     This base class behaves just like the native filesystem.
 158
 159     """
 160
 161     # imp interface
 162     def get_suffixes(self): return imp.get_suffixes()
 163     def new_module(self, name): return imp.new_module(name)
 164     def is_builtin(self, name): return imp.is_builtin(name)
 165     def init_builtin(self, name): return imp.init_builtin(name)
 166     def is_frozen(self, name): return imp.is_frozen(name)
 167     def init_frozen(self, name): return imp.init_frozen(name)
 168     def get_frozen_object(self, name): return imp.get_frozen_object(name)
 169     def load_source(self, name, filename, file=None):
 170         return imp.load_source(name, filename, file)
 171     def load_compiled(self, name, filename, file=None):
 172         return imp.load_compiled(name, filename, file)
 173     def load_dynamic(self, name, filename, file=None):
 174         return imp.load_dynamic(name, filename, file)
 175     def load_package(self, name, filename, file=None):
 176         return imp.load_module(name, file, filename, ("", "", PKG_DIRECTORY))
 177
 178     def add_module(self, name):
 179         d = self.modules_dict()
 180         if name in d: return d[name]
 181         d[name] = m = self.new_module(name)
 182         return m
 183
 184     # sys interface
 185     def modules_dict(self): return sys.modules
 186     def default_path(self): return sys.path
 187
 188     def path_split(self, x): return os.path.split(x)
 189     def path_join(self, x, y): return os.path.join(x, y)
 190     def path_isabs(self, x): return os.path.isabs(x)
 191     # etc.
 192
 193     def path_exists(self, x): return os.path.exists(x)
 194     def path_isdir(self, x): return os.path.isdir(x)
 195     def path_isfile(self, x): return os.path.isfile(x)
 196     def path_islink(self, x): return os.path.islink(x)
 197     # etc.
 198
 199     def openfile(self, *x): return open(*x)
 200     openfile_error = IOError
 201     def listdir(self, x): return os.listdir(x)
 202     listdir_error = os.error
 203     # etc.
 204
 205
 206 class ModuleLoader(BasicModuleLoader):
 207
 208     """Default module loader; uses file system hooks.
 209
 210     By defining suitable hooks, you might be able to load modules from
 211     other sources than the file system, e.g. from compressed or
 212     encrypted files, tar files or (if you're brave!) URLs.
 213
 214     """
 215
 216     def __init__(self, hooks = None, verbose = VERBOSE):
 217         BasicModuleLoader.__init__(self, verbose)
 218         self.hooks = hooks or Hooks(verbose)
 219
 220     def default_path(self):
 221         return self.hooks.default_path()
 222
 223     def modules_dict(self):
 224         return self.hooks.modules_dict()
 225
 226     def get_hooks(self):
 227         return self.hooks
 228
 229     def set_hooks(self, hooks):
 230         self.hooks = hooks
 231
 232     def find_builtin_module(self, name):
 233         # XXX frozen packages?
 234         if self.hooks.is_builtin(name):
 235             return None, '', ('', '', BUILTIN_MODULE)
 236         if self.hooks.is_frozen(name):
 237             return None, '', ('', '', FROZEN_MODULE)
 238         return None
 239
 240     def find_module_in_dir(self, name, dir, allow_packages=1):
 241         if dir is None:
 242             return self.find_builtin_module(name)
 243         if allow_packages:
 244             fullname = self.hooks.path_join(dir, name)
 245             if self.hooks.path_isdir(fullname):
 246                 stuff = self.find_module_in_dir("__init__", fullname, 0)
 247                 if stuff:
 248                     file = stuff[0]
 249                     if file: file.close()
 250                     return None, fullname, ('', '', PKG_DIRECTORY)
 251         for info in self.hooks.get_suffixes():
 252             suff, mode, type = info
 253             fullname = self.hooks.path_join(dir, name+suff)
 254             try:
 255                 fp = self.hooks.openfile(fullname, mode)
 256                 return fp, fullname, info
 257             except self.hooks.openfile_error:
 258                 pass
 259         return None
 260
 261     def load_module(self, name, stuff):
 262         file, filename, info = stuff
 263         (suff, mode, type) = info
 264         try:
 265             if type == BUILTIN_MODULE:
 266                 return self.hooks.init_builtin(name)
 267             if type == FROZEN_MODULE:
 268                 return self.hooks.init_frozen(name)
 269             if type == C_EXTENSION:
 270                 m = self.hooks.load_dynamic(name, filename, file)
 271             elif type == PY_SOURCE:
 272                 m = self.hooks.load_source(name, filename, file)
 273             elif type == PY_COMPILED:
 274                 m = self.hooks.load_compiled(name, filename, file)
 275             elif type == PKG_DIRECTORY:
 276                 m = self.hooks.load_package(name, filename, file)
 277             else:
 278                 raise ImportError, "Unrecognized module type (%r) for %s" % \
 279                       (type, name)
 280         finally:
 281             if file: file.close()
 282         m.__file__ = filename
 283         return m
 284
 285
 286 class FancyModuleLoader(ModuleLoader):
 287
 288     """Fancy module loader -- parses and execs the code itself."""
 289
 290     def load_module(self, name, stuff):
 291         file, filename, (suff, mode, type) = stuff
 292         realfilename = filename
 293         path = None
 294
 295         if type == PKG_DIRECTORY:
 296             initstuff = self.find_module_in_dir("__init__", filename, 0)
 297             if not initstuff:
 298                 raise ImportError, "No __init__ module in package %s" % name
 299             initfile, initfilename, initinfo = initstuff
 300             initsuff, initmode, inittype = initinfo
 301             if inittype not in (PY_COMPILED, PY_SOURCE):
 302                 if initfile: initfile.close()
 303                 raise ImportError, \
 304                     "Bad type (%r) for __init__ module in package %s" % (
 305                     inittype, name)
 306             path = [filename]
 307             file = initfile
 308             realfilename = initfilename
 309             type = inittype
 310
 311         if type == FROZEN_MODULE:
 312             code = self.hooks.get_frozen_object(name)
 313         elif type == PY_COMPILED:
 314             import marshal
 315             file.seek(8)
 316             code = marshal.load(file)
 317         elif type == PY_SOURCE:
 318             data = file.read()
 319             code = compile(data, realfilename, 'exec')
 320         else:
 321             return ModuleLoader.load_module(self, name, stuff)
 322
 323         m = self.hooks.add_module(name)
 324         if path:
 325             m.__path__ = path
 326         m.__file__ = filename
 327         try:
 328             exec code in m.__dict__
 329         except:
 330             d = self.hooks.modules_dict()
 331             if name in d:
 332                 del d[name]
 333             raise
 334         return m
 335
 336
 337 class BasicModuleImporter(_Verbose):
 338
 339     """Basic module importer; uses module loader.
 340
 341     This provides basic import facilities but no package imports.
 342
 343     """
 344
 345     def __init__(self, loader = None, verbose = VERBOSE):
 346         _Verbose.__init__(self, verbose)
 347         self.loader = loader or ModuleLoader(None, verbose)
 348         self.modules = self.loader.modules_dict()
 349
 350     def get_loader(self):
 351         return self.loader
 352
 353     def set_loader(self, loader):
 354         self.loader = loader
 355
 356     def get_hooks(self):
 357         return self.loader.get_hooks()
 358
 359     def set_hooks(self, hooks):
 360         return self.loader.set_hooks(hooks)
 361
 362     def import_module(self, name, globals={}, locals={}, fromlist=[]):
 363         name = str(name)
 364         if name in self.modules:
 365             return self.modules[name] # Fast path
 366         stuff = self.loader.find_module(name)
 367         if not stuff:
 368             raise ImportError, "No module named %s" % name
 369         return self.loader.load_module(name, stuff)
 370
 371     def reload(self, module, path = None):
 372         name = str(module.__name__)
 373         stuff = self.loader.find_module(name, path)
 374         if not stuff:
 375             raise ImportError, "Module %s not found for reload" % name
 376         return self.loader.load_module(name, stuff)
 377
 378     def unload(self, module):
 379         del self.modules[str(module.__name__)]
 380         # XXX Should this try to clear the module's namespace?
 381
 382     def install(self):
 383         self.save_import_module = __builtin__.__import__
 384         self.save_reload = __builtin__.reload
 385         if not hasattr(__builtin__, 'unload'):
 386             __builtin__.unload = None
 387         self.save_unload = __builtin__.unload
 388         __builtin__.__import__ = self.import_module
 389         __builtin__.reload = self.reload
 390         __builtin__.unload = self.unload
 391
 392     def uninstall(self):
 393         __builtin__.__import__ = self.save_import_module
 394         __builtin__.reload = self.save_reload
 395         __builtin__.unload = self.save_unload
 396         if not __builtin__.unload:
 397             del __builtin__.unload
 398
 399
 400 class ModuleImporter(BasicModuleImporter):
 401
 402     """A module importer that supports packages."""
 403
 404     def import_module(self, name, globals=None, locals=None, fromlist=None,
 405                       level=-1):
 406         parent = self.determine_parent(globals, level)
 407         q, tail = self.find_head_package(parent, str(name))
 408         m = self.load_tail(q, tail)
 409         if not fromlist:
 410             return q
 411         if hasattr(m, "__path__"):
 412             self.ensure_fromlist(m, fromlist)
 413         return m
 414
 415     def determine_parent(self, globals, level=-1):
 416         if not globals or not level:
 417             return None
 418         pkgname = globals.get('__package__')
 419         if pkgname is not None:
 420             if not pkgname and level > 0:
 421                 raise ValueError, 'Attempted relative import in non-package'
 422         else:
 423             # __package__ not set, figure it out and set it
 424             modname = globals.get('__name__')
 425             if modname is None:
 426                 return None
 427             if "__path__" in globals:
 428                 # __path__ is set so modname is already the package name
 429                 pkgname = modname
 430             else:
 431                 # normal module, work out package name if any
 432                 if '.' not in modname:
 433                     if level > 0:
 434                         raise ValueError, ('Attempted relative import in '
 435                                            'non-package')
 436                     globals['__package__'] = None
 437                     return None
 438                 pkgname = modname.rpartition('.')[0]
 439             globals['__package__'] = pkgname
 440         if level > 0:
 441             dot = len(pkgname)
 442             for x in range(level, 1, -1):
 443                 try:
 444                     dot = pkgname.rindex('.', 0, dot)
 445                 except ValueError:
 446                     raise ValueError('attempted relative import beyond '
 447                                      'top-level package')
 448             pkgname = pkgname[:dot]
 449         try:
 450             return sys.modules[pkgname]
 451         except KeyError:
 452             if level < 1:
 453                 warn("Parent module '%s' not found while handling "
 454                      "absolute import" % pkgname, RuntimeWarning, 1)
 455                 return None
 456             else:
 457                 raise SystemError, ("Parent module '%s' not loaded, cannot "
 458                                     "perform relative import" % pkgname)
 459
 460     def find_head_package(self, parent, name):
 461         if '.' in name:
 462             i = name.find('.')
 463             head = name[:i]
 464             tail = name[i+1:]
 465         else:
 466             head = name
 467             tail = ""
 468         if parent:
 469             qname = "%s.%s" % (parent.__name__, head)
 470         else:
 471             qname = head
 472         q = self.import_it(head, qname, parent)
 473         if q: return q, tail
 474         if parent:
 475             qname = head
 476             parent = None
 477             q = self.import_it(head, qname, parent)
 478             if q: return q, tail
 479         raise ImportError, "No module named '%s'" % qname
 480
 481     def load_tail(self, q, tail):
 482         m = q
 483         while tail:
 484             i = tail.find('.')
 485             if i < 0: i = len(tail)
 486             head, tail = tail[:i], tail[i+1:]
 487             mname = "%s.%s" % (m.__name__, head)
 488             m = self.import_it(head, mname, m)
 489             if not m:
 490                 raise ImportError, "No module named '%s'" % mname
 491         return m
 492
 493     def ensure_fromlist(self, m, fromlist, recursive=0):
 494         for sub in fromlist:
 495             if sub == "*":
 496                 if not recursive:
 497                     try:
 498                         all = m.__all__
 499                     except AttributeError:
 500                         pass
 501                     else:
 502                         self.ensure_fromlist(m, all, 1)
 503                 continue
 504             if sub != "*" and not hasattr(m, sub):
 505                 subname = "%s.%s" % (m.__name__, sub)
 506                 submod = self.import_it(sub, subname, m)
 507                 if not submod:
 508                     raise ImportError, "No module named '%s'" % subname
 509
 510     def import_it(self, partname, fqname, parent, force_load=0):
 511         if not partname:
 512             # completely empty module name should only happen in
 513             # 'from . import' or __import__("")
 514             return parent
 515         if not force_load:
 516             try:
 517                 return self.modules[fqname]
 518             except KeyError:
 519                 pass
 520         try:
 521             path = parent and parent.__path__
 522         except AttributeError:
 523             return None
 524         partname = str(partname)
 525         stuff = self.loader.find_module(partname, path)
 526         if not stuff:
 527             return None
 528         fqname = str(fqname)
 529         m = self.loader.load_module(fqname, stuff)
 530         if parent:
 531             setattr(parent, partname, m)
 532         return m
 533
 534     def reload(self, module):
 535         name = str(module.__name__)
 536         if '.' not in name:
 537             return self.import_it(name, name, None, force_load=1)
 538         i = name.rfind('.')
 539         pname = name[:i]
 540         parent = self.modules[pname]
 541         return self.import_it(name[i+1:], name, parent, force_load=1)
 542
 543
 544 default_importer = None
 545 current_importer = None
 546
 547 def install(importer = None):
 548     global current_importer
 549     current_importer = importer or default_importer or ModuleImporter()
 550     current_importer.install()
 551
 552 def uninstall():
 553     global current_importer
 554     current_importer.uninstall()