Using FS mtime to reload non recursive cache.
[pyTivo.git] / Cheetah / ImportManager.py
blob3ff1dbe2ff2ab06fa1f9f7fb51dd855b0a05fc38
1 #!/usr/bin/env python
2 # $Id: ImportManager.py,v 1.5 2006/01/27 19:00:20 tavis_rudd Exp $
4 """Provides an emulator/replacement for Python's standard import system.
6 @@TR: Be warned that Import Hooks are in the deepest, darkest corner of Python's
7 jungle. If you need to start hacking with this, be prepared to get lost for a
8 while. Also note, this module predates the newstyle import hooks in Python 2.3
9 http://www.python.org/peps/pep-0302.html.
12 This is a hacked/documented version of Gordon McMillan's iu.py. I have:
14 - made it a little less terse
16 - added docstrings and explanatations
18 - standardized the variable naming scheme
20 - reorganized the code layout to enhance readability
22 Meta-Data
23 ================================================================================
24 Author: Tavis Rudd <tavis@damnsimple.com> based on Gordon McMillan's iu.py
25 License: This software is released for unlimited distribution under the
26 terms of the MIT license. See the LICENSE file.
27 Version: $Revision: 1.5 $
28 Start Date: 2001/03/30
29 Last Revision Date: $Date: 2006/01/27 19:00:20 $
30 """
31 __author__ = "Tavis Rudd <tavis@damnsimple.com>"
32 __revision__ = "$Revision: 1.5 $"[11:-2]
34 ##################################################
35 ## DEPENDENCIES
37 import sys
38 import imp
39 import marshal
41 ##################################################
42 ## CONSTANTS & GLOBALS
44 try:
45 True,False
46 except NameError:
47 True, False = (1==1),(1==0)
49 _installed = False
51 STRINGTYPE = type('')
53 # _globalOwnerTypes is defined at the bottom of this file
55 _os_stat = _os_path_join = _os_getcwd = _os_path_dirname = None
57 ##################################################
58 ## FUNCTIONS
60 def _os_bootstrap():
61 """Set up 'os' module replacement functions for use during import bootstrap."""
63 names = sys.builtin_module_names
65 join = dirname = None
66 if 'posix' in names:
67 sep = '/'
68 from posix import stat, getcwd
69 elif 'nt' in names:
70 sep = '\\'
71 from nt import stat, getcwd
72 elif 'dos' in names:
73 sep = '\\'
74 from dos import stat, getcwd
75 elif 'os2' in names:
76 sep = '\\'
77 from os2 import stat, getcwd
78 elif 'mac' in names:
79 from mac import stat, getcwd
80 def join(a, b):
81 if a == '':
82 return b
83 path = s
84 if ':' not in a:
85 a = ':' + a
86 if a[-1:] != ':':
87 a = a + ':'
88 return a + b
89 else:
90 raise ImportError, 'no os specific module found'
92 if join is None:
93 def join(a, b, sep=sep):
94 if a == '':
95 return b
96 lastchar = a[-1:]
97 if lastchar == '/' or lastchar == sep:
98 return a + b
99 return a + sep + b
101 if dirname is None:
102 def dirname(a, sep=sep):
103 for i in range(len(a)-1, -1, -1):
104 c = a[i]
105 if c == '/' or c == sep:
106 return a[:i]
107 return ''
109 global _os_stat
110 _os_stat = stat
112 global _os_path_join
113 _os_path_join = join
115 global _os_path_dirname
116 _os_path_dirname = dirname
118 global _os_getcwd
119 _os_getcwd = getcwd
121 _os_bootstrap()
123 def packageName(s):
124 for i in range(len(s)-1, -1, -1):
125 if s[i] == '.':
126 break
127 else:
128 return ''
129 return s[:i]
131 def nameSplit(s):
132 rslt = []
133 i = j = 0
134 for j in range(len(s)):
135 if s[j] == '.':
136 rslt.append(s[i:j])
137 i = j+1
138 if i < len(s):
139 rslt.append(s[i:])
140 return rslt
142 def getPathExt(fnm):
143 for i in range(len(fnm)-1, -1, -1):
144 if fnm[i] == '.':
145 return fnm[i:]
146 return ''
148 def pathIsDir(pathname):
149 "Local replacement for os.path.isdir()."
150 try:
151 s = _os_stat(pathname)
152 except OSError:
153 return None
154 return (s[0] & 0170000) == 0040000
156 def getDescr(fnm):
157 ext = getPathExt(fnm)
158 for (suffix, mode, typ) in imp.get_suffixes():
159 if suffix == ext:
160 return (suffix, mode, typ)
162 ##################################################
163 ## CLASSES
165 class Owner:
167 """An Owner does imports from a particular piece of turf That is, there's
168 an Owner for each thing on sys.path There are owners for directories and
169 .pyz files. There could be owners for zip files, or even URLs. A
170 shadowpath (a dictionary mapping the names in sys.path to their owners) is
171 used so that sys.path (or a package's __path__) is still a bunch of strings,
174 def __init__(self, path):
175 self.path = path
177 def __str__(self):
178 return self.path
180 def getmod(self, nm):
181 return None
183 class DirOwner(Owner):
185 def __init__(self, path):
186 if path == '':
187 path = _os_getcwd()
188 if not pathIsDir(path):
189 raise ValueError, "%s is not a directory" % path
190 Owner.__init__(self, path)
192 def getmod(self, nm,
193 getsuffixes=imp.get_suffixes, loadco=marshal.loads, newmod=imp.new_module):
195 pth = _os_path_join(self.path, nm)
197 possibles = [(pth, 0, None)]
198 if pathIsDir(pth):
199 possibles.insert(0, (_os_path_join(pth, '__init__'), 1, pth))
200 py = pyc = None
201 for pth, ispkg, pkgpth in possibles:
202 for ext, mode, typ in getsuffixes():
203 attempt = pth+ext
204 try:
205 st = _os_stat(attempt)
206 except:
207 pass
208 else:
209 if typ == imp.C_EXTENSION:
210 fp = open(attempt, 'rb')
211 mod = imp.load_module(nm, fp, attempt, (ext, mode, typ))
212 mod.__file__ = attempt
213 return mod
214 elif typ == imp.PY_SOURCE:
215 py = (attempt, st)
216 else:
217 pyc = (attempt, st)
218 if py or pyc:
219 break
220 if py is None and pyc is None:
221 return None
222 while 1:
223 if pyc is None or py and pyc[1][8] < py[1][8]:
224 try:
225 co = compile(open(py[0], 'r').read()+'\n', py[0], 'exec')
226 break
227 except SyntaxError, e:
228 print "Invalid syntax in %s" % py[0]
229 print e.args
230 raise
231 elif pyc:
232 stuff = open(pyc[0], 'rb').read()
233 try:
234 co = loadco(stuff[8:])
235 break
236 except (ValueError, EOFError):
237 pyc = None
238 else:
239 return None
240 mod = newmod(nm)
241 mod.__file__ = co.co_filename
242 if ispkg:
243 mod.__path__ = [pkgpth]
244 subimporter = PathImportDirector(mod.__path__)
245 mod.__importsub__ = subimporter.getmod
246 mod.__co__ = co
247 return mod
250 class ImportDirector(Owner):
251 """ImportDirectors live on the metapath There's one for builtins, one for
252 frozen modules, and one for sys.path Windows gets one for modules gotten
253 from the Registry Mac would have them for PY_RESOURCE modules etc. A
254 generalization of Owner - their concept of 'turf' is broader"""
256 pass
258 class BuiltinImportDirector(ImportDirector):
259 """Directs imports of builtin modules"""
260 def __init__(self):
261 self.path = 'Builtins'
263 def getmod(self, nm, isbuiltin=imp.is_builtin):
264 if isbuiltin(nm):
265 mod = imp.load_module(nm, None, nm, ('','',imp.C_BUILTIN))
266 return mod
267 return None
269 class FrozenImportDirector(ImportDirector):
270 """Directs imports of frozen modules"""
272 def __init__(self):
273 self.path = 'FrozenModules'
275 def getmod(self, nm,
276 isFrozen=imp.is_frozen, loadMod=imp.load_module):
277 if isFrozen(nm):
278 mod = loadMod(nm, None, nm, ('','',imp.PY_FROZEN))
279 if hasattr(mod, '__path__'):
280 mod.__importsub__ = lambda name, pname=nm, owner=self: owner.getmod(pname+'.'+name)
281 return mod
282 return None
285 class RegistryImportDirector(ImportDirector):
286 """Directs imports of modules stored in the Windows Registry"""
288 def __init__(self):
289 self.path = "WindowsRegistry"
290 self.map = {}
291 try:
292 import win32api
293 ## import win32con
294 except ImportError:
295 pass
296 else:
297 HKEY_CURRENT_USER = -2147483647
298 HKEY_LOCAL_MACHINE = -2147483646
299 KEY_ALL_ACCESS = 983103
300 subkey = r"Software\Python\PythonCore\%s\Modules" % sys.winver
301 for root in (HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE):
302 try:
303 hkey = win32api.RegOpenKeyEx(root, subkey, 0, KEY_ALL_ACCESS)
304 except:
305 pass
306 else:
307 numsubkeys, numvalues, lastmodified = win32api.RegQueryInfoKey(hkey)
308 for i in range(numsubkeys):
309 subkeyname = win32api.RegEnumKey(hkey, i)
310 hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, KEY_ALL_ACCESS)
311 val = win32api.RegQueryValueEx(hskey, '')
312 desc = getDescr(val[0])
313 self.map[subkeyname] = (val[0], desc)
314 hskey.Close()
315 hkey.Close()
316 break
318 def getmod(self, nm):
319 stuff = self.map.get(nm)
320 if stuff:
321 fnm, desc = stuff
322 fp = open(fnm, 'rb')
323 mod = imp.load_module(nm, fp, fnm, desc)
324 mod.__file__ = fnm
325 return mod
326 return None
328 class PathImportDirector(ImportDirector):
329 """Directs imports of modules stored on the filesystem."""
331 def __init__(self, pathlist=None, importers=None, ownertypes=None):
332 if pathlist is None:
333 self.path = sys.path
334 else:
335 self.path = pathlist
336 if ownertypes == None:
337 self._ownertypes = _globalOwnerTypes
338 else:
339 self._ownertypes = ownertypes
340 if importers:
341 self._shadowPath = importers
342 else:
343 self._shadowPath = {}
344 self._inMakeOwner = False
345 self._building = {}
347 def getmod(self, nm):
348 mod = None
349 for thing in self.path:
350 if type(thing) is STRINGTYPE:
351 owner = self._shadowPath.get(thing, -1)
352 if owner == -1:
353 owner = self._shadowPath[thing] = self._makeOwner(thing)
354 if owner:
355 mod = owner.getmod(nm)
356 else:
357 mod = thing.getmod(nm)
358 if mod:
359 break
360 return mod
362 def _makeOwner(self, path):
363 if self._building.get(path):
364 return None
365 self._building[path] = 1
366 owner = None
367 for klass in self._ownertypes:
368 try:
369 # this may cause an import, which may cause recursion
370 # hence the protection
371 owner = klass(path)
372 except:
373 pass
374 else:
375 break
376 del self._building[path]
377 return owner
379 #=================ImportManager============================#
380 # The one-and-only ImportManager
381 # ie, the builtin import
383 UNTRIED = -1
385 class ImportManager:
386 # really the equivalent of builtin import
387 def __init__(self):
388 self.metapath = [
389 BuiltinImportDirector(),
390 FrozenImportDirector(),
391 RegistryImportDirector(),
392 PathImportDirector()
394 self.threaded = 0
395 self.rlock = None
396 self.locker = None
397 self.setThreaded()
399 def setThreaded(self):
400 thread = sys.modules.get('thread', None)
401 if thread and not self.threaded:
402 self.threaded = 1
403 self.rlock = thread.allocate_lock()
404 self._get_ident = thread.get_ident
406 def install(self):
407 import __builtin__
408 __builtin__.__import__ = self.importHook
409 __builtin__.reload = self.reloadHook
411 def importHook(self, name, globals=None, locals=None, fromlist=None):
412 # first see if we could be importing a relative name
413 #print "importHook(%s, %s, locals, %s)" % (name, globals['__name__'], fromlist)
414 _sys_modules_get = sys.modules.get
415 contexts = [None]
416 if globals:
417 importernm = globals.get('__name__', '')
418 if importernm:
419 if hasattr(_sys_modules_get(importernm), '__path__'):
420 contexts.insert(0,importernm)
421 else:
422 pkgnm = packageName(importernm)
423 if pkgnm:
424 contexts.insert(0,pkgnm)
425 # so contexts is [pkgnm, None] or just [None]
426 # now break the name being imported up so we get:
427 # a.b.c -> [a, b, c]
428 nmparts = nameSplit(name)
429 _self_doimport = self.doimport
430 threaded = self.threaded
431 for context in contexts:
432 ctx = context
433 for i in range(len(nmparts)):
434 nm = nmparts[i]
435 #print " importHook trying %s in %s" % (nm, ctx)
436 if ctx:
437 fqname = ctx + '.' + nm
438 else:
439 fqname = nm
440 if threaded:
441 self._acquire()
442 mod = _sys_modules_get(fqname, UNTRIED)
443 if mod is UNTRIED:
444 mod = _self_doimport(nm, ctx, fqname)
445 if threaded:
446 self._release()
447 if mod:
448 ctx = fqname
449 else:
450 break
451 else:
452 # no break, point i beyond end
453 i = i + 1
454 if i:
455 break
457 if i<len(nmparts):
458 if ctx and hasattr(sys.modules[ctx], nmparts[i]):
459 #print "importHook done with %s %s %s (case 1)" % (name, globals['__name__'], fromlist)
460 return sys.modules[nmparts[0]]
461 del sys.modules[fqname]
462 raise ImportError, "No module named %s" % fqname
463 if fromlist is None:
464 #print "importHook done with %s %s %s (case 2)" % (name, globals['__name__'], fromlist)
465 if context:
466 return sys.modules[context+'.'+nmparts[0]]
467 return sys.modules[nmparts[0]]
468 bottommod = sys.modules[ctx]
469 if hasattr(bottommod, '__path__'):
470 fromlist = list(fromlist)
471 i = 0
472 while i < len(fromlist):
473 nm = fromlist[i]
474 if nm == '*':
475 fromlist[i:i+1] = list(getattr(bottommod, '__all__', []))
476 if i >= len(fromlist):
477 break
478 nm = fromlist[i]
479 i = i + 1
480 if not hasattr(bottommod, nm):
481 if self.threaded:
482 self._acquire()
483 mod = self.doimport(nm, ctx, ctx+'.'+nm)
484 if self.threaded:
485 self._release()
486 if not mod:
487 raise ImportError, "%s not found in %s" % (nm, ctx)
488 #print "importHook done with %s %s %s (case 3)" % (name, globals['__name__'], fromlist)
489 return bottommod
491 def doimport(self, nm, parentnm, fqname):
492 # Not that nm is NEVER a dotted name at this point
493 #print "doimport(%s, %s, %s)" % (nm, parentnm, fqname)
494 if parentnm:
495 parent = sys.modules[parentnm]
496 if hasattr(parent, '__path__'):
497 importfunc = getattr(parent, '__importsub__', None)
498 if not importfunc:
499 subimporter = PathImportDirector(parent.__path__)
500 importfunc = parent.__importsub__ = subimporter.getmod
501 mod = importfunc(nm)
502 if mod:
503 setattr(parent, nm, mod)
504 else:
505 #print "..parent not a package"
506 return None
507 else:
508 # now we're dealing with an absolute import
509 for director in self.metapath:
510 mod = director.getmod(nm)
511 if mod:
512 break
513 if mod:
514 mod.__name__ = fqname
515 sys.modules[fqname] = mod
516 if hasattr(mod, '__co__'):
517 co = mod.__co__
518 del mod.__co__
519 exec co in mod.__dict__
520 if fqname == 'thread' and not self.threaded:
521 ## print "thread detected!"
522 self.setThreaded()
523 else:
524 sys.modules[fqname] = None
525 #print "..found %s" % mod
526 return mod
528 def reloadHook(self, mod):
529 fqnm = mod.__name__
530 nm = nameSplit(fqnm)[-1]
531 parentnm = packageName(fqnm)
532 newmod = self.doimport(nm, parentnm, fqnm)
533 mod.__dict__.update(newmod.__dict__)
534 ## return newmod
536 def _acquire(self):
537 if self.rlock.locked():
538 if self.locker == self._get_ident():
539 self.lockcount = self.lockcount + 1
540 ## print "_acquire incrementing lockcount to", self.lockcount
541 return
542 self.rlock.acquire()
543 self.locker = self._get_ident()
544 self.lockcount = 0
545 ## print "_acquire first time!"
547 def _release(self):
548 if self.lockcount:
549 self.lockcount = self.lockcount - 1
550 ## print "_release decrementing lockcount to", self.lockcount
551 else:
552 self.rlock.release()
553 ## print "_release releasing lock!"
556 ##################################################
557 ## MORE CONSTANTS & GLOBALS
559 _globalOwnerTypes = [
560 DirOwner,
561 Owner,