Cheetah/ImportManager.py

   1 #!/usr/bin/env python
   2 # $Id: ImportManager.py,v 1.6 2007/04/03 01:56:24 tavis_rudd Exp $
   3
   4 """Provides an emulator/replacement for Python's standard import system.
   5
   6 @@TR: Be warned that Import Hooks are in the deepest, darkest corner of Python's
   7 jungle.  If you need to start hacking with this, be prepared to get lost for a
   8 while. Also note, this module predates the newstyle import hooks in Python 2.3
   9 http://www.python.org/peps/pep-0302.html.
  10
  11
  12 This is a hacked/documented version of Gordon McMillan's iu.py. I have:
  13
  14   - made it a little less terse
  15
  16   - added docstrings and explanatations
  17
  18   - standardized the variable naming scheme
  19
  20   - reorganized the code layout to enhance readability
  21
  22 Meta-Data
  23 ================================================================================
  24 Author: Tavis Rudd <tavis@damnsimple.com>  based on Gordon McMillan's iu.py
  25 License: This software is released for unlimited distribution under the
  26          terms of the MIT license.  See the LICENSE file.
  27 Version: $Revision: 1.6 $
  28 Start Date: 2001/03/30
  29 Last Revision Date: $Date: 2007/04/03 01:56:24 $
  30 """
  31 __author__ = "Tavis Rudd <tavis@damnsimple.com>"
  32 __revision__ = "$Revision: 1.6 $"[11:-2]
  33
  34 ##################################################
  35 ## DEPENDENCIES
  36
  37 import sys
  38 import imp
  39 import marshal
  40
  41 ##################################################
  42 ## CONSTANTS & GLOBALS
  43
  44 try:
  45     True,False
  46 except NameError:
  47     True, False = (1==1),(1==0)
  48
  49 _installed = False
  50
  51 STRINGTYPE = type('')
  52
  53 # _globalOwnerTypes is defined at the bottom of this file
  54
  55 _os_stat = _os_path_join = _os_getcwd = _os_path_dirname = None
  56
  57 ##################################################
  58 ## FUNCTIONS
  59
  60 def _os_bootstrap():
  61     """Set up 'os' module replacement functions for use during import bootstrap."""
  62
  63     names = sys.builtin_module_names
  64
  65     join = dirname = None
  66     if 'posix' in names:
  67         sep = '/'
  68         from posix import stat, getcwd
  69     elif 'nt' in names:
  70         sep = '\\'
  71         from nt import stat, getcwd
  72     elif 'dos' in names:
  73         sep = '\\'
  74         from dos import stat, getcwd
  75     elif 'os2' in names:
  76         sep = '\\'
  77         from os2 import stat, getcwd
  78     elif 'mac' in names:
  79         from mac import stat, getcwd
  80         def join(a, b):
  81             if a == '':
  82                 return b
  83             if ':' not in a:
  84                 a = ':' + a
  85             if a[-1:] != ':':
  86                 a = a + ':'
  87             return a + b
  88     else:
  89         raise ImportError, 'no os specific module found'
  90
  91     if join is None:
  92         def join(a, b, sep=sep):
  93             if a == '':
  94                 return b
  95             lastchar = a[-1:]
  96             if lastchar == '/' or lastchar == sep:
  97                 return a + b
  98             return a + sep + b
  99
 100     if dirname is None:
 101         def dirname(a, sep=sep):
 102             for i in range(len(a)-1, -1, -1):
 103                 c = a[i]
 104                 if c == '/' or c == sep:
 105                     return a[:i]
 106             return ''
 107
 108     global _os_stat
 109     _os_stat = stat
 110
 111     global _os_path_join
 112     _os_path_join = join
 113
 114     global _os_path_dirname
 115     _os_path_dirname = dirname
 116
 117     global _os_getcwd
 118     _os_getcwd = getcwd
 119
 120 _os_bootstrap()
 121
 122 def packageName(s):
 123     for i in range(len(s)-1, -1, -1):
 124         if s[i] == '.':
 125             break
 126     else:
 127         return ''
 128     return s[:i]
 129
 130 def nameSplit(s):
 131     rslt = []
 132     i = j = 0
 133     for j in range(len(s)):
 134         if s[j] == '.':
 135             rslt.append(s[i:j])
 136             i = j+1
 137     if i < len(s):
 138         rslt.append(s[i:])
 139     return rslt
 140
 141 def getPathExt(fnm):
 142     for i in range(len(fnm)-1, -1, -1):
 143         if fnm[i] == '.':
 144             return fnm[i:]
 145     return ''
 146
 147 def pathIsDir(pathname):
 148     "Local replacement for os.path.isdir()."
 149     try:
 150         s = _os_stat(pathname)
 151     except OSError:
 152         return None
 153     return (s[0] & 0170000) == 0040000
 154
 155 def getDescr(fnm):
 156     ext = getPathExt(fnm)
 157     for (suffix, mode, typ) in imp.get_suffixes():
 158         if suffix == ext:
 159             return (suffix, mode, typ)
 160
 161 ##################################################
 162 ## CLASSES
 163
 164 class Owner:
 165
 166     """An Owner does imports from a particular piece of turf That is, there's
 167     an Owner for each thing on sys.path There are owners for directories and
 168     .pyz files.  There could be owners for zip files, or even URLs.  A
 169     shadowpath (a dictionary mapping the names in sys.path to their owners) is
 170     used so that sys.path (or a package's __path__) is still a bunch of strings,
 171     """
 172
 173     def __init__(self, path):
 174         self.path = path
 175
 176     def __str__(self):
 177         return self.path
 178
 179     def getmod(self, nm):
 180         return None
 181
 182 class DirOwner(Owner):
 183
 184     def __init__(self, path):
 185         if path == '':
 186             path = _os_getcwd()
 187         if not pathIsDir(path):
 188             raise ValueError, "%s is not a directory" % path
 189         Owner.__init__(self, path)
 190
 191     def getmod(self, nm,
 192                getsuffixes=imp.get_suffixes, loadco=marshal.loads, newmod=imp.new_module):
 193
 194         pth =  _os_path_join(self.path, nm)
 195
 196         possibles = [(pth, 0, None)]
 197         if pathIsDir(pth):
 198             possibles.insert(0, (_os_path_join(pth, '__init__'), 1, pth))
 199         py = pyc = None
 200         for pth, ispkg, pkgpth in possibles:
 201             for ext, mode, typ in getsuffixes():
 202                 attempt = pth+ext
 203                 try:
 204                     st = _os_stat(attempt)
 205                 except:
 206                     pass
 207                 else:
 208                     if typ == imp.C_EXTENSION:
 209                         fp = open(attempt, 'rb')
 210                         mod = imp.load_module(nm, fp, attempt, (ext, mode, typ))
 211                         mod.__file__ = attempt
 212                         return mod
 213                     elif typ == imp.PY_SOURCE:
 214                         py = (attempt, st)
 215                     else:
 216                         pyc = (attempt, st)
 217             if py or pyc:
 218                 break
 219         if py is None and pyc is None:
 220             return None
 221         while 1:
 222             if pyc is None or py and pyc[1][8] < py[1][8]:
 223                 try:
 224                     co = compile(open(py[0], 'r').read()+'\n', py[0], 'exec')
 225                     break
 226                 except SyntaxError, e:
 227                     print "Invalid syntax in %s" % py[0]
 228                     print e.args
 229                     raise
 230             elif pyc:
 231                 stuff = open(pyc[0], 'rb').read()
 232                 try:
 233                     co = loadco(stuff[8:])
 234                     break
 235                 except (ValueError, EOFError):
 236                     pyc = None
 237             else:
 238                 return None
 239         mod = newmod(nm)
 240         mod.__file__ = co.co_filename
 241         if ispkg:
 242             mod.__path__ = [pkgpth]
 243             subimporter = PathImportDirector(mod.__path__)
 244             mod.__importsub__ = subimporter.getmod
 245         mod.__co__ = co
 246         return mod
 247
 248
 249 class ImportDirector(Owner):
 250     """ImportDirectors live on the metapath There's one for builtins, one for
 251     frozen modules, and one for sys.path Windows gets one for modules gotten
 252     from the Registry Mac would have them for PY_RESOURCE modules etc.  A
 253     generalization of Owner - their concept of 'turf' is broader"""
 254
 255     pass
 256
 257 class BuiltinImportDirector(ImportDirector):
 258     """Directs imports of builtin modules"""
 259     def __init__(self):
 260         self.path = 'Builtins'
 261
 262     def getmod(self, nm, isbuiltin=imp.is_builtin):
 263         if isbuiltin(nm):
 264             mod = imp.load_module(nm, None, nm, ('','',imp.C_BUILTIN))
 265             return mod
 266         return None
 267
 268 class FrozenImportDirector(ImportDirector):
 269     """Directs imports of frozen modules"""
 270
 271     def __init__(self):
 272         self.path = 'FrozenModules'
 273
 274     def getmod(self, nm,
 275                isFrozen=imp.is_frozen, loadMod=imp.load_module):
 276         if isFrozen(nm):
 277             mod = loadMod(nm, None, nm, ('','',imp.PY_FROZEN))
 278             if hasattr(mod, '__path__'):
 279                 mod.__importsub__ = lambda name, pname=nm, owner=self: owner.getmod(pname+'.'+name)
 280             return mod
 281         return None
 282
 283
 284 class RegistryImportDirector(ImportDirector):
 285     """Directs imports of modules stored in the Windows Registry"""
 286
 287     def __init__(self):
 288         self.path = "WindowsRegistry"
 289         self.map = {}
 290         try:
 291             import win32api
 292             ## import win32con
 293         except ImportError:
 294             pass
 295         else:
 296             HKEY_CURRENT_USER = -2147483647
 297             HKEY_LOCAL_MACHINE = -2147483646
 298             KEY_ALL_ACCESS = 983103
 299             subkey = r"Software\Python\PythonCore\%s\Modules" % sys.winver
 300             for root in (HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE):
 301                 try:
 302                     hkey = win32api.RegOpenKeyEx(root, subkey, 0, KEY_ALL_ACCESS)
 303                 except:
 304                     pass
 305                 else:
 306                     numsubkeys, numvalues, lastmodified = win32api.RegQueryInfoKey(hkey)
 307                     for i in range(numsubkeys):
 308                         subkeyname = win32api.RegEnumKey(hkey, i)
 309                         hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, KEY_ALL_ACCESS)
 310                         val = win32api.RegQueryValueEx(hskey, '')
 311                         desc = getDescr(val[0])
 312                         self.map[subkeyname] = (val[0], desc)
 313                         hskey.Close()
 314                     hkey.Close()
 315                     break
 316
 317     def getmod(self, nm):
 318         stuff = self.map.get(nm)
 319         if stuff:
 320             fnm, desc = stuff
 321             fp = open(fnm, 'rb')
 322             mod = imp.load_module(nm, fp, fnm, desc)
 323             mod.__file__ = fnm
 324             return mod
 325         return None
 326
 327 class PathImportDirector(ImportDirector):
 328     """Directs imports of modules stored on the filesystem."""
 329
 330     def __init__(self, pathlist=None, importers=None, ownertypes=None):
 331         if pathlist is None:
 332             self.path = sys.path
 333         else:
 334             self.path = pathlist
 335         if ownertypes == None:
 336             self._ownertypes = _globalOwnerTypes
 337         else:
 338             self._ownertypes = ownertypes
 339         if importers:
 340             self._shadowPath = importers
 341         else:
 342             self._shadowPath = {}
 343         self._inMakeOwner = False
 344         self._building = {}
 345
 346     def getmod(self, nm):
 347         mod = None
 348         for thing in self.path:
 349             if type(thing) is STRINGTYPE:
 350                 owner = self._shadowPath.get(thing, -1)
 351                 if owner == -1:
 352                     owner = self._shadowPath[thing] = self._makeOwner(thing)
 353                 if owner:
 354                     mod = owner.getmod(nm)
 355             else:
 356                 mod = thing.getmod(nm)
 357             if mod:
 358                 break
 359         return mod
 360
 361     def _makeOwner(self, path):
 362         if self._building.get(path):
 363             return None
 364         self._building[path] = 1
 365         owner = None
 366         for klass in self._ownertypes:
 367             try:
 368                 # this may cause an import, which may cause recursion
 369                 # hence the protection
 370                 owner = klass(path)
 371             except:
 372                 pass
 373             else:
 374                 break
 375         del self._building[path]
 376         return owner
 377
 378 #=================ImportManager============================#
 379 # The one-and-only ImportManager
 380 # ie, the builtin import
 381
 382 UNTRIED = -1
 383
 384 class ImportManager:
 385     # really the equivalent of builtin import
 386     def __init__(self):
 387         self.metapath = [
 388             BuiltinImportDirector(),
 389             FrozenImportDirector(),
 390             RegistryImportDirector(),
 391             PathImportDirector()
 392         ]
 393         self.threaded = 0
 394         self.rlock = None
 395         self.locker = None
 396         self.setThreaded()
 397
 398     def setThreaded(self):
 399         thread = sys.modules.get('thread', None)
 400         if thread and not self.threaded:
 401             self.threaded = 1
 402             self.rlock = thread.allocate_lock()
 403             self._get_ident = thread.get_ident
 404
 405     def install(self):
 406         import __builtin__
 407         __builtin__.__import__ = self.importHook
 408         __builtin__.reload = self.reloadHook
 409
 410     def importHook(self, name, globals=None, locals=None, fromlist=None):
 411         # first see if we could be importing a relative name
 412         #print "importHook(%s, %s, locals, %s)" % (name, globals['__name__'], fromlist)
 413         _sys_modules_get = sys.modules.get
 414         contexts = [None]
 415         if globals:
 416             importernm = globals.get('__name__', '')
 417             if importernm:
 418                 if hasattr(_sys_modules_get(importernm), '__path__'):
 419                     contexts.insert(0,importernm)
 420                 else:
 421                     pkgnm = packageName(importernm)
 422                     if pkgnm:
 423                         contexts.insert(0,pkgnm)
 424         # so contexts is [pkgnm, None] or just [None]
 425         # now break the name being imported up so we get:
 426         # a.b.c -> [a, b, c]
 427         nmparts = nameSplit(name)
 428         _self_doimport = self.doimport
 429         threaded = self.threaded
 430         for context in contexts:
 431             ctx = context
 432             for i in range(len(nmparts)):
 433                 nm = nmparts[i]
 434                 #print " importHook trying %s in %s" % (nm, ctx)
 435                 if ctx:
 436                     fqname = ctx + '.' + nm
 437                 else:
 438                     fqname = nm
 439                 if threaded:
 440                     self._acquire()
 441                 mod = _sys_modules_get(fqname, UNTRIED)
 442                 if mod is UNTRIED:
 443                     mod = _self_doimport(nm, ctx, fqname)
 444                 if threaded:
 445                     self._release()
 446                 if mod:
 447                     ctx = fqname
 448                 else:
 449                     break
 450             else:
 451                 # no break, point i beyond end
 452                 i = i + 1
 453             if i:
 454                 break
 455
 456         if i<len(nmparts):
 457             if ctx and hasattr(sys.modules[ctx], nmparts[i]):
 458                 #print "importHook done with %s %s %s (case 1)" % (name, globals['__name__'], fromlist)
 459                 return sys.modules[nmparts[0]]
 460             del sys.modules[fqname]
 461             raise ImportError, "No module named %s" % fqname
 462         if fromlist is None:
 463             #print "importHook done with %s %s %s (case 2)" % (name, globals['__name__'], fromlist)
 464             if context:
 465                 return sys.modules[context+'.'+nmparts[0]]
 466             return sys.modules[nmparts[0]]
 467         bottommod = sys.modules[ctx]
 468         if hasattr(bottommod, '__path__'):
 469             fromlist = list(fromlist)
 470             i = 0
 471             while i < len(fromlist):
 472                 nm = fromlist[i]
 473                 if nm == '*':
 474                     fromlist[i:i+1] = list(getattr(bottommod, '__all__', []))
 475                     if i >= len(fromlist):
 476                         break
 477                     nm = fromlist[i]
 478                 i = i + 1
 479                 if not hasattr(bottommod, nm):
 480                     if self.threaded:
 481                         self._acquire()
 482                     mod = self.doimport(nm, ctx, ctx+'.'+nm)
 483                     if self.threaded:
 484                         self._release()
 485                     if not mod:
 486                         raise ImportError, "%s not found in %s" % (nm, ctx)
 487         #print "importHook done with %s %s %s (case 3)" % (name, globals['__name__'], fromlist)
 488         return bottommod
 489
 490     def doimport(self, nm, parentnm, fqname):
 491         # Not that nm is NEVER a dotted name at this point
 492         #print "doimport(%s, %s, %s)" % (nm, parentnm, fqname)
 493         if parentnm:
 494             parent = sys.modules[parentnm]
 495             if hasattr(parent, '__path__'):
 496                 importfunc = getattr(parent, '__importsub__', None)
 497                 if not importfunc:
 498                     subimporter = PathImportDirector(parent.__path__)
 499                     importfunc = parent.__importsub__ = subimporter.getmod
 500                 mod = importfunc(nm)
 501                 if mod:
 502                     setattr(parent, nm, mod)
 503             else:
 504                 #print "..parent not a package"
 505                 return None
 506         else:
 507             # now we're dealing with an absolute import
 508             for director in self.metapath:
 509                 mod = director.getmod(nm)
 510                 if mod:
 511                     break
 512         if mod:
 513             mod.__name__ = fqname
 514             sys.modules[fqname] = mod
 515             if hasattr(mod, '__co__'):
 516                 co = mod.__co__
 517                 del mod.__co__
 518                 exec co in mod.__dict__
 519             if fqname == 'thread' and not self.threaded:
 520 ##                print "thread detected!"
 521                 self.setThreaded()
 522         else:
 523             sys.modules[fqname] = None
 524         #print "..found %s" % mod
 525         return mod
 526
 527     def reloadHook(self, mod):
 528         fqnm = mod.__name__
 529         nm = nameSplit(fqnm)[-1]
 530         parentnm = packageName(fqnm)
 531         newmod = self.doimport(nm, parentnm, fqnm)
 532         mod.__dict__.update(newmod.__dict__)
 533 ##        return newmod
 534
 535     def _acquire(self):
 536         if self.rlock.locked():
 537             if self.locker == self._get_ident():
 538                 self.lockcount = self.lockcount + 1
 539 ##                print "_acquire incrementing lockcount to", self.lockcount
 540                 return
 541         self.rlock.acquire()
 542         self.locker = self._get_ident()
 543         self.lockcount = 0
 544 ##        print "_acquire first time!"
 545
 546     def _release(self):
 547         if self.lockcount:
 548             self.lockcount = self.lockcount - 1
 549 ##            print "_release decrementing lockcount to", self.lockcount
 550         else:
 551             self.rlock.release()
 552 ##            print "_release releasing lock!"
 553
 554
 555 ##################################################
 556 ## MORE CONSTANTS & GLOBALS
 557
 558 _globalOwnerTypes = [
 559     DirOwner,
 560     Owner,
 561 ]