Cheetah/ImportManager.py

   1 #!/usr/bin/env python
   2 # $Id: ImportManager.py,v 1.5 2006/01/27 19:00:20 tavis_rudd Exp $
   3
   4 """Provides an emulator/replacement for Python's standard import system.
   5
   6 @@TR: Be warned that Import Hooks are in the deepest, darkest corner of Python's
   7 jungle.  If you need to start hacking with this, be prepared to get lost for a
   8 while. Also note, this module predates the newstyle import hooks in Python 2.3
   9 http://www.python.org/peps/pep-0302.html.
  10
  11
  12 This is a hacked/documented version of Gordon McMillan's iu.py. I have:
  13
  14   - made it a little less terse
  15
  16   - added docstrings and explanatations
  17
  18   - standardized the variable naming scheme
  19
  20   - reorganized the code layout to enhance readability
  21
  22 Meta-Data
  23 ================================================================================
  24 Author: Tavis Rudd <tavis@damnsimple.com>  based on Gordon McMillan's iu.py
  25 License: This software is released for unlimited distribution under the
  26          terms of the MIT license.  See the LICENSE file.
  27 Version: $Revision: 1.5 $
  28 Start Date: 2001/03/30
  29 Last Revision Date: $Date: 2006/01/27 19:00:20 $
  30 """
  31 __author__ = "Tavis Rudd <tavis@damnsimple.com>"
  32 __revision__ = "$Revision: 1.5 $"[11:-2]
  33
  34 ##################################################
  35 ## DEPENDENCIES
  36
  37 import sys
  38 import imp
  39 import marshal
  40
  41 ##################################################
  42 ## CONSTANTS & GLOBALS
  43
  44 try:
  45     True,False
  46 except NameError:
  47     True, False = (1==1),(1==0)
  48
  49 _installed = False
  50
  51 STRINGTYPE = type('')
  52
  53 # _globalOwnerTypes is defined at the bottom of this file
  54
  55 _os_stat = _os_path_join = _os_getcwd = _os_path_dirname = None
  56
  57 ##################################################
  58 ## FUNCTIONS
  59
  60 def _os_bootstrap():
  61     """Set up 'os' module replacement functions for use during import bootstrap."""
  62
  63     names = sys.builtin_module_names
  64
  65     join = dirname = None
  66     if 'posix' in names:
  67         sep = '/'
  68         from posix import stat, getcwd
  69     elif 'nt' in names:
  70         sep = '\\'
  71         from nt import stat, getcwd
  72     elif 'dos' in names:
  73         sep = '\\'
  74         from dos import stat, getcwd
  75     elif 'os2' in names:
  76         sep = '\\'
  77         from os2 import stat, getcwd
  78     elif 'mac' in names:
  79         from mac import stat, getcwd
  80         def join(a, b):
  81             if a == '':
  82                 return b
  83             path = s
  84             if ':' not in a:
  85                 a = ':' + a
  86             if a[-1:] != ':':
  87                 a = a + ':'
  88             return a + b
  89     else:
  90         raise ImportError, 'no os specific module found'
  91
  92     if join is None:
  93         def join(a, b, sep=sep):
  94             if a == '':
  95                 return b
  96             lastchar = a[-1:]
  97             if lastchar == '/' or lastchar == sep:
  98                 return a + b
  99             return a + sep + b
 100
 101     if dirname is None:
 102         def dirname(a, sep=sep):
 103             for i in range(len(a)-1, -1, -1):
 104                 c = a[i]
 105                 if c == '/' or c == sep:
 106                     return a[:i]
 107             return ''
 108
 109     global _os_stat
 110     _os_stat = stat
 111
 112     global _os_path_join
 113     _os_path_join = join
 114
 115     global _os_path_dirname
 116     _os_path_dirname = dirname
 117
 118     global _os_getcwd
 119     _os_getcwd = getcwd
 120
 121 _os_bootstrap()
 122
 123 def packageName(s):
 124     for i in range(len(s)-1, -1, -1):
 125         if s[i] == '.':
 126             break
 127     else:
 128         return ''
 129     return s[:i]
 130
 131 def nameSplit(s):
 132     rslt = []
 133     i = j = 0
 134     for j in range(len(s)):
 135         if s[j] == '.':
 136             rslt.append(s[i:j])
 137             i = j+1
 138     if i < len(s):
 139         rslt.append(s[i:])
 140     return rslt
 141
 142 def getPathExt(fnm):
 143     for i in range(len(fnm)-1, -1, -1):
 144         if fnm[i] == '.':
 145             return fnm[i:]
 146     return ''
 147
 148 def pathIsDir(pathname):
 149     "Local replacement for os.path.isdir()."
 150     try:
 151         s = _os_stat(pathname)
 152     except OSError:
 153         return None
 154     return (s[0] & 0170000) == 0040000
 155
 156 def getDescr(fnm):
 157     ext = getPathExt(fnm)
 158     for (suffix, mode, typ) in imp.get_suffixes():
 159         if suffix == ext:
 160             return (suffix, mode, typ)
 161
 162 ##################################################
 163 ## CLASSES
 164
 165 class Owner:
 166
 167     """An Owner does imports from a particular piece of turf That is, there's
 168     an Owner for each thing on sys.path There are owners for directories and
 169     .pyz files.  There could be owners for zip files, or even URLs.  A
 170     shadowpath (a dictionary mapping the names in sys.path to their owners) is
 171     used so that sys.path (or a package's __path__) is still a bunch of strings,
 172     """
 173
 174     def __init__(self, path):
 175         self.path = path
 176
 177     def __str__(self):
 178         return self.path
 179
 180     def getmod(self, nm):
 181         return None
 182
 183 class DirOwner(Owner):
 184
 185     def __init__(self, path):
 186         if path == '':
 187             path = _os_getcwd()
 188         if not pathIsDir(path):
 189             raise ValueError, "%s is not a directory" % path
 190         Owner.__init__(self, path)
 191
 192     def getmod(self, nm,
 193                getsuffixes=imp.get_suffixes, loadco=marshal.loads, newmod=imp.new_module):
 194
 195         pth =  _os_path_join(self.path, nm)
 196
 197         possibles = [(pth, 0, None)]
 198         if pathIsDir(pth):
 199             possibles.insert(0, (_os_path_join(pth, '__init__'), 1, pth))
 200         py = pyc = None
 201         for pth, ispkg, pkgpth in possibles:
 202             for ext, mode, typ in getsuffixes():
 203                 attempt = pth+ext
 204                 try:
 205                     st = _os_stat(attempt)
 206                 except:
 207                     pass
 208                 else:
 209                     if typ == imp.C_EXTENSION:
 210                         fp = open(attempt, 'rb')
 211                         mod = imp.load_module(nm, fp, attempt, (ext, mode, typ))
 212                         mod.__file__ = attempt
 213                         return mod
 214                     elif typ == imp.PY_SOURCE:
 215                         py = (attempt, st)
 216                     else:
 217                         pyc = (attempt, st)
 218             if py or pyc:
 219                 break
 220         if py is None and pyc is None:
 221             return None
 222         while 1:
 223             if pyc is None or py and pyc[1][8] < py[1][8]:
 224                 try:
 225                     co = compile(open(py[0], 'r').read()+'\n', py[0], 'exec')
 226                     break
 227                 except SyntaxError, e:
 228                     print "Invalid syntax in %s" % py[0]
 229                     print e.args
 230                     raise
 231             elif pyc:
 232                 stuff = open(pyc[0], 'rb').read()
 233                 try:
 234                     co = loadco(stuff[8:])
 235                     break
 236                 except (ValueError, EOFError):
 237                     pyc = None
 238             else:
 239                 return None
 240         mod = newmod(nm)
 241         mod.__file__ = co.co_filename
 242         if ispkg:
 243             mod.__path__ = [pkgpth]
 244             subimporter = PathImportDirector(mod.__path__)
 245             mod.__importsub__ = subimporter.getmod
 246         mod.__co__ = co
 247         return mod
 248
 249
 250 class ImportDirector(Owner):
 251     """ImportDirectors live on the metapath There's one for builtins, one for
 252     frozen modules, and one for sys.path Windows gets one for modules gotten
 253     from the Registry Mac would have them for PY_RESOURCE modules etc.  A
 254     generalization of Owner - their concept of 'turf' is broader"""
 255
 256     pass
 257
 258 class BuiltinImportDirector(ImportDirector):
 259     """Directs imports of builtin modules"""
 260     def __init__(self):
 261         self.path = 'Builtins'
 262
 263     def getmod(self, nm, isbuiltin=imp.is_builtin):
 264         if isbuiltin(nm):
 265             mod = imp.load_module(nm, None, nm, ('','',imp.C_BUILTIN))
 266             return mod
 267         return None
 268
 269 class FrozenImportDirector(ImportDirector):
 270     """Directs imports of frozen modules"""
 271
 272     def __init__(self):
 273         self.path = 'FrozenModules'
 274
 275     def getmod(self, nm,
 276                isFrozen=imp.is_frozen, loadMod=imp.load_module):
 277         if isFrozen(nm):
 278             mod = loadMod(nm, None, nm, ('','',imp.PY_FROZEN))
 279             if hasattr(mod, '__path__'):
 280                 mod.__importsub__ = lambda name, pname=nm, owner=self: owner.getmod(pname+'.'+name)
 281             return mod
 282         return None
 283
 284
 285 class RegistryImportDirector(ImportDirector):
 286     """Directs imports of modules stored in the Windows Registry"""
 287
 288     def __init__(self):
 289         self.path = "WindowsRegistry"
 290         self.map = {}
 291         try:
 292             import win32api
 293             ## import win32con
 294         except ImportError:
 295             pass
 296         else:
 297             HKEY_CURRENT_USER = -2147483647
 298             HKEY_LOCAL_MACHINE = -2147483646
 299             KEY_ALL_ACCESS = 983103
 300             subkey = r"Software\Python\PythonCore\%s\Modules" % sys.winver
 301             for root in (HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE):
 302                 try:
 303                     hkey = win32api.RegOpenKeyEx(root, subkey, 0, KEY_ALL_ACCESS)
 304                 except:
 305                     pass
 306                 else:
 307                     numsubkeys, numvalues, lastmodified = win32api.RegQueryInfoKey(hkey)
 308                     for i in range(numsubkeys):
 309                         subkeyname = win32api.RegEnumKey(hkey, i)
 310                         hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, KEY_ALL_ACCESS)
 311                         val = win32api.RegQueryValueEx(hskey, '')
 312                         desc = getDescr(val[0])
 313                         self.map[subkeyname] = (val[0], desc)
 314                         hskey.Close()
 315                     hkey.Close()
 316                     break
 317
 318     def getmod(self, nm):
 319         stuff = self.map.get(nm)
 320         if stuff:
 321             fnm, desc = stuff
 322             fp = open(fnm, 'rb')
 323             mod = imp.load_module(nm, fp, fnm, desc)
 324             mod.__file__ = fnm
 325             return mod
 326         return None
 327
 328 class PathImportDirector(ImportDirector):
 329     """Directs imports of modules stored on the filesystem."""
 330
 331     def __init__(self, pathlist=None, importers=None, ownertypes=None):
 332         if pathlist is None:
 333             self.path = sys.path
 334         else:
 335             self.path = pathlist
 336         if ownertypes == None:
 337             self._ownertypes = _globalOwnerTypes
 338         else:
 339             self._ownertypes = ownertypes
 340         if importers:
 341             self._shadowPath = importers
 342         else:
 343             self._shadowPath = {}
 344         self._inMakeOwner = False
 345         self._building = {}
 346
 347     def getmod(self, nm):
 348         mod = None
 349         for thing in self.path:
 350             if type(thing) is STRINGTYPE:
 351                 owner = self._shadowPath.get(thing, -1)
 352                 if owner == -1:
 353                     owner = self._shadowPath[thing] = self._makeOwner(thing)
 354                 if owner:
 355                     mod = owner.getmod(nm)
 356             else:
 357                 mod = thing.getmod(nm)
 358             if mod:
 359                 break
 360         return mod
 361
 362     def _makeOwner(self, path):
 363         if self._building.get(path):
 364             return None
 365         self._building[path] = 1
 366         owner = None
 367         for klass in self._ownertypes:
 368             try:
 369                 # this may cause an import, which may cause recursion
 370                 # hence the protection
 371                 owner = klass(path)
 372             except:
 373                 pass
 374             else:
 375                 break
 376         del self._building[path]
 377         return owner
 378
 379 #=================ImportManager============================#
 380 # The one-and-only ImportManager
 381 # ie, the builtin import
 382
 383 UNTRIED = -1
 384
 385 class ImportManager:
 386     # really the equivalent of builtin import
 387     def __init__(self):
 388         self.metapath = [
 389             BuiltinImportDirector(),
 390             FrozenImportDirector(),
 391             RegistryImportDirector(),
 392             PathImportDirector()
 393         ]
 394         self.threaded = 0
 395         self.rlock = None
 396         self.locker = None
 397         self.setThreaded()
 398
 399     def setThreaded(self):
 400         thread = sys.modules.get('thread', None)
 401         if thread and not self.threaded:
 402             self.threaded = 1
 403             self.rlock = thread.allocate_lock()
 404             self._get_ident = thread.get_ident
 405
 406     def install(self):
 407         import __builtin__
 408         __builtin__.__import__ = self.importHook
 409         __builtin__.reload = self.reloadHook
 410
 411     def importHook(self, name, globals=None, locals=None, fromlist=None):
 412         # first see if we could be importing a relative name
 413         #print "importHook(%s, %s, locals, %s)" % (name, globals['__name__'], fromlist)
 414         _sys_modules_get = sys.modules.get
 415         contexts = [None]
 416         if globals:
 417             importernm = globals.get('__name__', '')
 418             if importernm:
 419                 if hasattr(_sys_modules_get(importernm), '__path__'):
 420                     contexts.insert(0,importernm)
 421                 else:
 422                     pkgnm = packageName(importernm)
 423                     if pkgnm:
 424                         contexts.insert(0,pkgnm)
 425         # so contexts is [pkgnm, None] or just [None]
 426         # now break the name being imported up so we get:
 427         # a.b.c -> [a, b, c]
 428         nmparts = nameSplit(name)
 429         _self_doimport = self.doimport
 430         threaded = self.threaded
 431         for context in contexts:
 432             ctx = context
 433             for i in range(len(nmparts)):
 434                 nm = nmparts[i]
 435                 #print " importHook trying %s in %s" % (nm, ctx)
 436                 if ctx:
 437                     fqname = ctx + '.' + nm
 438                 else:
 439                     fqname = nm
 440                 if threaded:
 441                     self._acquire()
 442                 mod = _sys_modules_get(fqname, UNTRIED)
 443                 if mod is UNTRIED:
 444                     mod = _self_doimport(nm, ctx, fqname)
 445                 if threaded:
 446                     self._release()
 447                 if mod:
 448                     ctx = fqname
 449                 else:
 450                     break
 451             else:
 452                 # no break, point i beyond end
 453                 i = i + 1
 454             if i:
 455                 break
 456
 457         if i<len(nmparts):
 458             if ctx and hasattr(sys.modules[ctx], nmparts[i]):
 459                 #print "importHook done with %s %s %s (case 1)" % (name, globals['__name__'], fromlist)
 460                 return sys.modules[nmparts[0]]
 461             del sys.modules[fqname]
 462             raise ImportError, "No module named %s" % fqname
 463         if fromlist is None:
 464             #print "importHook done with %s %s %s (case 2)" % (name, globals['__name__'], fromlist)
 465             if context:
 466                 return sys.modules[context+'.'+nmparts[0]]
 467             return sys.modules[nmparts[0]]
 468         bottommod = sys.modules[ctx]
 469         if hasattr(bottommod, '__path__'):
 470             fromlist = list(fromlist)
 471             i = 0
 472             while i < len(fromlist):
 473                 nm = fromlist[i]
 474                 if nm == '*':
 475                     fromlist[i:i+1] = list(getattr(bottommod, '__all__', []))
 476                     if i >= len(fromlist):
 477                         break
 478                     nm = fromlist[i]
 479                 i = i + 1
 480                 if not hasattr(bottommod, nm):
 481                     if self.threaded:
 482                         self._acquire()
 483                     mod = self.doimport(nm, ctx, ctx+'.'+nm)
 484                     if self.threaded:
 485                         self._release()
 486                     if not mod:
 487                         raise ImportError, "%s not found in %s" % (nm, ctx)
 488         #print "importHook done with %s %s %s (case 3)" % (name, globals['__name__'], fromlist)
 489         return bottommod
 490
 491     def doimport(self, nm, parentnm, fqname):
 492         # Not that nm is NEVER a dotted name at this point
 493         #print "doimport(%s, %s, %s)" % (nm, parentnm, fqname)
 494         if parentnm:
 495             parent = sys.modules[parentnm]
 496             if hasattr(parent, '__path__'):
 497                 importfunc = getattr(parent, '__importsub__', None)
 498                 if not importfunc:
 499                     subimporter = PathImportDirector(parent.__path__)
 500                     importfunc = parent.__importsub__ = subimporter.getmod
 501                 mod = importfunc(nm)
 502                 if mod:
 503                     setattr(parent, nm, mod)
 504             else:
 505                 #print "..parent not a package"
 506                 return None
 507         else:
 508             # now we're dealing with an absolute import
 509             for director in self.metapath:
 510                 mod = director.getmod(nm)
 511                 if mod:
 512                     break
 513         if mod:
 514             mod.__name__ = fqname
 515             sys.modules[fqname] = mod
 516             if hasattr(mod, '__co__'):
 517                 co = mod.__co__
 518                 del mod.__co__
 519                 exec co in mod.__dict__
 520             if fqname == 'thread' and not self.threaded:
 521 ##                print "thread detected!"
 522                 self.setThreaded()
 523         else:
 524             sys.modules[fqname] = None
 525         #print "..found %s" % mod
 526         return mod
 527
 528     def reloadHook(self, mod):
 529         fqnm = mod.__name__
 530         nm = nameSplit(fqnm)[-1]
 531         parentnm = packageName(fqnm)
 532         newmod = self.doimport(nm, parentnm, fqnm)
 533         mod.__dict__.update(newmod.__dict__)
 534 ##        return newmod
 535
 536     def _acquire(self):
 537         if self.rlock.locked():
 538             if self.locker == self._get_ident():
 539                 self.lockcount = self.lockcount + 1
 540 ##                print "_acquire incrementing lockcount to", self.lockcount
 541                 return
 542         self.rlock.acquire()
 543         self.locker = self._get_ident()
 544         self.lockcount = 0
 545 ##        print "_acquire first time!"
 546
 547     def _release(self):
 548         if self.lockcount:
 549             self.lockcount = self.lockcount - 1
 550 ##            print "_release decrementing lockcount to", self.lockcount
 551         else:
 552             self.rlock.release()
 553 ##            print "_release releasing lock!"
 554
 555
 556 ##################################################
 557 ## MORE CONSTANTS & GLOBALS
 558
 559 _globalOwnerTypes = [
 560     DirOwner,
 561     Owner,
 562 ]