5 ImportManager Manage the import process
7 Importer Base class for replacing standard import functions
8 BuiltinImporter Emulate the import mechanism for builtin and frozen modules
12 from warnings
import warnpy3k
13 warnpy3k("the imputil module has been removed in Python 3.0", stacklevel
=2)
16 # note: avoid importing non-builtin modules
17 import imp
### not available in Jython?
21 # for the DirectoryImporter
25 __all__
= ["ImportManager","Importer","BuiltinImporter"]
27 _StringType
= type('')
28 _ModuleType
= type(sys
) ### doesn't work in Jython...
31 "Manage the import process."
33 def install(self
, namespace
=vars(__builtin__
)):
34 "Install this ImportManager into the specified namespace."
36 if isinstance(namespace
, _ModuleType
):
37 namespace
= vars(namespace
)
39 # Note: we have no notion of "chaining"
41 # Record the previous import hook, then install our own.
42 self
.previous_importer
= namespace
['__import__']
43 self
.namespace
= namespace
44 namespace
['__import__'] = self
._import
_hook
47 #namespace['reload'] = self._reload_hook
50 "Restore the previous import mechanism."
51 self
.namespace
['__import__'] = self
.previous_importer
53 def add_suffix(self
, suffix
, importFunc
):
54 assert hasattr(importFunc
, '__call__')
55 self
.fs_imp
.add_suffix(suffix
, importFunc
)
57 ######################################################################
62 clsFilesystemImporter
= None
64 def __init__(self
, fs_imp
=None):
65 # we're definitely going to be importing something in the future,
66 # so let's just load the OS-related facilities.
70 # This is the Importer that we use for grabbing stuff from the
71 # filesystem. It defines one more method (import_from_dir) for our use.
73 cls
= self
.clsFilesystemImporter
or _FilesystemImporter
77 # Initialize the set of suffixes that we recognize and import.
78 # The default will import dynamic-load modules first, followed by
79 # .py files (or a .py file's cached bytecode)
80 for desc
in imp
.get_suffixes():
81 if desc
[2] == imp
.C_EXTENSION
:
82 self
.add_suffix(desc
[0],
83 DynLoadSuffixImporter(desc
).import_file
)
84 self
.add_suffix('.py', py_suffix_importer
)
86 def _import_hook(self
, fqname
, globals=None, locals=None, fromlist
=None):
87 """Python calls this hook to locate and import a module."""
89 parts
= fqname
.split('.')
91 # determine the context of this import
92 parent
= self
._determine
_import
_context
(globals)
94 # if there is a parent, then its importer should manage this import
96 module
= parent
.__importer
__._do
_import
(parent
, parts
, fromlist
)
100 # has the top module already been imported?
102 top_module
= sys
.modules
[parts
[0]]
105 # look for the topmost module
106 top_module
= self
._import
_top
_module
(parts
[0])
108 # the topmost module wasn't found at all.
109 raise ImportError, 'No module named ' + fqname
111 # fast-path simple imports
116 if not top_module
.__dict
__.get('__ispkg__'):
117 # __ispkg__ isn't defined (the module was not imported by us),
120 # In the former case, there is no way that we could import
121 # sub-modules that occur in the fromlist (but we can't raise an
122 # error because it may just be names) because we don't know how
123 # to deal with packages that were imported by other systems.
125 # In the latter case (__ispkg__ == 0), there can't be any sub-
126 # modules present, so we can just return.
128 # In both cases, since len(parts) == 1, the top_module is also
129 # the "bottom" which is the defined return when a fromlist
133 importer
= top_module
.__dict
__.get('__importer__')
135 return importer
._finish
_import
(top_module
, parts
[1:], fromlist
)
137 # Grrr, some people "import os.path" or do "from os.path import ..."
138 if len(parts
) == 2 and hasattr(top_module
, parts
[1]):
140 return getattr(top_module
, parts
[1])
144 # If the importer does not exist, then we have to bail. A missing
145 # importer means that something else imported the module, and we have
146 # no knowledge of how to get sub-modules out of the thing.
147 raise ImportError, 'No module named ' + fqname
149 def _determine_import_context(self
, globals):
150 """Returns the context in which a module should be imported.
152 The context could be a loaded (package) module and the imported module
153 will be looked for within that package. The context could also be None,
154 meaning there is no context -- the module should be looked for as a
158 if not globals or not globals.get('__importer__'):
159 # globals does not refer to one of our modules or packages. That
160 # implies there is no relative import context (as far as we are
161 # concerned), and it should just pick it off the standard path.
164 # The globals refer to a module or package of ours. It will define
165 # the context of the new import. Get the module/package fqname.
166 parent_fqname
= globals['__name__']
168 # if a package is performing the import, then return itself (imports
169 # refer to pkg contents)
170 if globals['__ispkg__']:
171 parent
= sys
.modules
[parent_fqname
]
172 assert globals is parent
.__dict
__
175 i
= parent_fqname
.rfind('.')
177 # a module outside of a package has no particular import context
181 # if a module in a package is performing the import, then return the
182 # package (imports refer to siblings)
183 parent_fqname
= parent_fqname
[:i
]
184 parent
= sys
.modules
[parent_fqname
]
185 assert parent
.__name
__ == parent_fqname
188 def _import_top_module(self
, name
):
189 # scan sys.path looking for a location in the filesystem that contains
190 # the module, or an Importer object that can import the module.
191 for item
in sys
.path
:
192 if isinstance(item
, _StringType
):
193 module
= self
.fs_imp
.import_from_dir(item
, name
)
195 module
= item
.import_top(name
)
200 def _reload_hook(self
, module
):
201 "Python calls this hook to reload a module."
203 # reloading of a module may or may not be possible (depending on the
204 # importer), but at least we can validate that it's ours to reload
205 importer
= module
.__dict
__.get('__importer__')
207 ### oops. now what...
210 # okay. it is using the imputil system, and we must delegate it, but
211 # we don't know what to do (yet)
212 ### we should blast the module dict and do another get_code(). need to
213 ### flesh this out and add proper docco...
214 raise SystemError, "reload not yet implemented"
218 "Base class for replacing standard import functions."
220 def import_top(self
, name
):
221 "Import a top-level module."
222 return self
._import
_one
(None, name
, name
)
224 ######################################################################
228 def _finish_import(self
, top
, parts
, fromlist
):
229 # if "a.b.c" was provided, then load the ".b.c" portion down from
230 # below the top-level module.
231 bottom
= self
._load
_tail
(top
, parts
)
233 # if the form is "import a.b.c", then return "a"
235 # no fromlist: return the top of the import tree
238 # the top module was imported by self.
240 # this means that the bottom module was also imported by self (just
241 # now, or in the past and we fetched it from sys.modules).
243 # since we imported/handled the bottom module, this means that we can
244 # also handle its fromlist (and reliably use __ispkg__).
246 # if the bottom node is a package, then (potentially) import some
249 # note: if it is not a package, then "fromlist" refers to names in
250 # the bottom module rather than modules.
251 # note: for a mix of names and modules in the fromlist, we will
252 # import all modules and insert those into the namespace of
253 # the package module. Python will pick up all fromlist names
254 # from the bottom (package) module; some will be modules that
255 # we imported and stored in the namespace, others are expected
256 # to be present already.
258 self
._import
_fromlist
(bottom
, fromlist
)
260 # if the form is "from a.b import c, d" then return "b"
263 def _import_one(self
, parent
, modname
, fqname
):
264 "Import a single module."
266 # has the module already been imported?
268 return sys
.modules
[fqname
]
272 # load the module's code, or fetch the module itself
273 result
= self
.get_code(parent
, modname
, fqname
)
277 module
= self
._process
_result
(result
, fqname
)
279 # insert the module into its parent
281 setattr(parent
, modname
, module
)
284 def _process_result(self
, result
, fqname
):
285 ispkg
, code
, values
= result
286 # did get_code() return an actual module? (rather than a code object)
287 is_module
= isinstance(code
, _ModuleType
)
289 # use the returned module, or create a new one to exec code into
293 module
= imp
.new_module(fqname
)
295 ### record packages a bit differently??
296 module
.__importer
__ = self
297 module
.__ispkg
__ = ispkg
299 # insert additional values into the module (before executing the code)
300 module
.__dict
__.update(values
)
302 # the module is almost ready... make it visible
303 sys
.modules
[fqname
] = module
305 # execute the code within the module's namespace
308 exec code
in module
.__dict
__
310 if fqname
in sys
.modules
:
311 del sys
.modules
[fqname
]
314 # fetch from sys.modules instead of returning module directly.
315 # also make module's __name__ agree with fqname, in case
316 # the "exec code in module.__dict__" played games on us.
317 module
= sys
.modules
[fqname
]
318 module
.__name
__ = fqname
321 def _load_tail(self
, m
, parts
):
322 """Import the rest of the modules, down from the top-level module.
324 Returns the last module in the dotted list of modules.
327 fqname
= "%s.%s" % (m
.__name
__, part
)
328 m
= self
._import
_one
(m
, part
, fqname
)
330 raise ImportError, "No module named " + fqname
333 def _import_fromlist(self
, package
, fromlist
):
334 'Import any sub-modules in the "from" list.'
336 # if '*' is present in the fromlist, then look for the '__all__'
337 # variable to find additional items (modules) to import.
339 fromlist
= list(fromlist
) + \
340 list(package
.__dict
__.get('__all__', []))
343 # if the name is already present, then don't try to import it (it
344 # might not be a module!).
345 if sub
!= '*' and not hasattr(package
, sub
):
346 subname
= "%s.%s" % (package
.__name
__, sub
)
347 submod
= self
._import
_one
(package
, sub
, subname
)
349 raise ImportError, "cannot import name " + subname
351 def _do_import(self
, parent
, parts
, fromlist
):
352 """Attempt to import the module relative to parent.
354 This method is used when the import context specifies that <self>
355 imported the parent module.
358 top_fqname
= parent
.__name
__ + '.' + top_name
359 top_module
= self
._import
_one
(parent
, top_name
, top_fqname
)
361 # this importer and parent could not find the module (relatively)
364 return self
._finish
_import
(top_module
, parts
[1:], fromlist
)
366 ######################################################################
368 # METHODS TO OVERRIDE
370 def get_code(self
, parent
, modname
, fqname
):
371 """Find and retrieve the code for the given module.
373 parent specifies a parent module to define a context for importing. It
374 may be None, indicating no particular context for the search.
376 modname specifies a single module (not dotted) within the parent.
378 fqname specifies the fully-qualified module name. This is a
379 (potentially) dotted name from the "root" of the module namespace
381 If there is no parent, then modname==fqname.
383 This method should return None, or a 3-tuple.
385 * If the module was not found, then None should be returned.
387 * The first item of the 2- or 3-tuple should be the integer 0 or 1,
388 specifying whether the module that was found is a package or not.
390 * The second item is the code object for the module (it will be
391 executed within the new module's namespace). This item can also
392 be a fully-loaded module object (e.g. loaded from a shared lib).
394 * The third item is a dictionary of name/value pairs that will be
395 inserted into new module before the code object is executed. This
396 is provided in case the module's code expects certain values (such
397 as where the module was found). When the second item is a module
398 object, then these names/values will be inserted *after* the module
399 has been loaded/initialized.
401 raise RuntimeError, "get_code not implemented"
404 ######################################################################
406 # Some handy stuff for the Importers
409 # byte-compiled file suffix character
410 _suffix_char
= __debug__
and 'c' or 'o'
412 # byte-compiled file suffix
413 _suffix
= '.py' + _suffix_char
415 def _compile(pathname
, timestamp
):
416 """Compile (and cache) a Python source file.
418 The file specified by <pathname> is compiled to a code object and
421 Presuming the appropriate privileges exist, the bytecodes will be
422 saved back to the filesystem for future imports. The source file's
423 modification timestamp must be provided as a Long value.
425 codestring
= open(pathname
, 'rU').read()
426 if codestring
and codestring
[-1] != '\n':
427 codestring
= codestring
+ '\n'
428 code
= __builtin__
.compile(codestring
, pathname
, 'exec')
430 # try to cache the compiled code
432 f
= open(pathname
+ _suffix_char
, 'wb')
437 f
.write(struct
.pack('<I', timestamp
))
438 marshal
.dump(code
, f
)
441 f
.write(imp
.get_magic())
446 _os_stat
= _os_path_join
= None
448 "Set up 'os' module replacement functions for use during import bootstrap."
450 names
= sys
.builtin_module_names
455 from posix
import stat
476 raise ImportError, 'no os specific module found'
479 def join(a
, b
, sep
=sep
):
483 if lastchar
== '/' or lastchar
== sep
:
493 def _os_path_isdir(pathname
):
494 "Local replacement for os.path.isdir()."
496 s
= _os_stat(pathname
)
499 return (s
.st_mode
& 0170000) == 0040000
501 def _timestamp(pathname
):
502 "Return the file modification time as a Long."
504 s
= _os_stat(pathname
)
507 return long(s
.st_mtime
)
510 ######################################################################
512 # Emulate the import mechanism for builtin and frozen modules
514 class BuiltinImporter(Importer
):
515 def get_code(self
, parent
, modname
, fqname
):
517 # these modules definitely do not occur within a package context
520 # look for the module
521 if imp
.is_builtin(modname
):
523 elif imp
.is_frozen(modname
):
529 # got it. now load and return it.
530 module
= imp
.load_module(modname
, None, modname
, ('', '', type))
531 return 0, module
, { }
534 ######################################################################
536 # Internal importer used for importing from the filesystem
538 class _FilesystemImporter(Importer
):
542 def add_suffix(self
, suffix
, importFunc
):
543 assert hasattr(importFunc
, '__call__')
544 self
.suffixes
.append((suffix
, importFunc
))
546 def import_from_dir(self
, dir, fqname
):
547 result
= self
._import
_pathname
(_os_path_join(dir, fqname
), fqname
)
549 return self
._process
_result
(result
, fqname
)
552 def get_code(self
, parent
, modname
, fqname
):
553 # This importer is never used with an empty parent. Its existence is
554 # private to the ImportManager. The ImportManager uses the
555 # import_from_dir() method to import top-level modules/packages.
556 # This method is only used when we look for a module within a package.
559 for submodule_path
in parent
.__path
__:
560 code
= self
._import
_pathname
(_os_path_join(submodule_path
, modname
), fqname
)
563 return self
._import
_pathname
(_os_path_join(parent
.__pkgdir
__, modname
),
566 def _import_pathname(self
, pathname
, fqname
):
567 if _os_path_isdir(pathname
):
568 result
= self
._import
_pathname
(_os_path_join(pathname
, '__init__'),
572 values
['__pkgdir__'] = pathname
573 values
['__path__'] = [ pathname
]
574 return 1, result
[1], values
577 for suffix
, importFunc
in self
.suffixes
:
578 filename
= pathname
+ suffix
580 finfo
= _os_stat(filename
)
584 return importFunc(filename
, finfo
, fqname
)
587 ######################################################################
589 # SUFFIX-BASED IMPORTERS
592 def py_suffix_importer(filename
, finfo
, fqname
):
593 file = filename
[:-3] + _suffix
594 t_py
= long(finfo
[8])
595 t_pyc
= _timestamp(file)
598 if t_pyc
is not None and t_pyc
>= t_py
:
600 if f
.read(4) == imp
.get_magic():
601 t
= struct
.unpack('<I', f
.read(4))[0]
603 code
= marshal
.load(f
)
607 code
= _compile(file, t_py
)
609 return 0, code
, { '__file__' : file }
611 class DynLoadSuffixImporter
:
612 def __init__(self
, desc
):
615 def import_file(self
, filename
, finfo
, fqname
):
616 fp
= open(filename
, self
.desc
[1])
617 module
= imp
.load_module(fqname
, fp
, filename
, self
.desc
)
618 module
.__file
__ = filename
619 return 0, module
, { }
622 ######################################################################
624 def _print_importers():
625 items
= sys
.modules
.items()
627 for name
, module
in items
:
629 print name
, module
.__dict
__.get('__importer__', '-- no importer')
631 print name
, '-- non-existent module'
634 ImportManager().install()
635 sys
.path
.insert(0, BuiltinImporter())
637 ######################################################################
643 # type(sys) is not a module in Jython. what to use instead?
644 # imp.C_EXTENSION is not in Jython. same for get_suffixes and new_module
648 # sys.modules['foo'] = sys
650 # ---- standard import mechanism
653 # <module 'sys' (built-in)>
655 # ---- revamped import mechanism
657 # >>> imputil._test_revamp()
660 # <module 'foo' from 'foo.py'>
664 # should BuiltinImporter exist in sys.path or hard-wired in ImportManager?
665 # need __path__ processing
667 # move chaining to a subclass [gjs: it's been nuked]
668 # deinstall should be possible
669 # query mechanism needed: is a specific Importer installed?
670 # py/pyc/pyo piping hooks to filter/process these files
672 # distutils importer hooked to list of standard Internet repositories
673 # module->file location mapper to speed FS-based imports
675 # keep chaining so that it can play nice with other import hooks
678 # push MAL's mapper into sys.path[0] as a cache (hard-coded for apps)
681 # need to change sys.* references for rexec environs
682 # need hook for MAL's walk-me-up import strategy, or Tim's absolute strategy
683 # watch out for sys.modules[...] is None
684 # flag to force absolute imports? (speeds _determine_import_context and
685 # checking for a relative module)
686 # insert names of archives into sys.path (see quote below)
687 # note: reload does NOT blast module dict
688 # shift import mechanisms and policies around; provide for hooks, overrides
690 # add get_source stuff
691 # get_topcode and get_subcode
692 # CRLF handling in _compile
693 # race condition in _compile
694 # refactoring of os.py to deal with _os_bootstrap problem
695 # any special handling to do for importing a module with a SyntaxError?
696 # (e.g. clean up the traceback)
697 # implement "domain" for path-type functionality using pkg namespace
698 # (rather than FS-names like __path__)
699 # don't use the word "private"... maybe "internal"
702 # Guido's comments on sys.path caching:
704 # We could cache this in a dictionary: the ImportManager can have a
705 # cache dict mapping pathnames to importer objects, and a separate
706 # method for coming up with an importer given a pathname that's not yet
707 # in the cache. The method should do a stat and/or look at the
708 # extension to decide which importer class to use; you can register new
709 # importer classes by registering a suffix or a Boolean function, plus a
710 # class. If you register a new importer class, the cache is zapped.
711 # The cache is independent from sys.path (but maintained per
712 # ImportManager instance) so that rearrangements of sys.path do the
713 # right thing. If a path is dropped from sys.path the corresponding
714 # cache entry is simply no longer used.
716 # My/Guido's comments on factoring ImportManager and Importer:
718 # > However, we still have a tension occurring here:
720 # > 1) implementing policy in ImportManager assists in single-point policy
721 # > changes for app/rexec situations
722 # > 2) implementing policy in Importer assists in package-private policy
723 # > changes for normal, operating conditions
725 # > I'll see if I can sort out a way to do this. Maybe the Importer class will
726 # > implement the methods (which can be overridden to change policy) by
727 # > delegating to ImportManager.
729 # Maybe also think about what kind of policies an Importer would be
730 # likely to want to change. I have a feeling that a lot of the code
731 # there is actually not so much policy but a *necessity* to get things
732 # working given the calling conventions for the __import__ hook: whether
733 # to return the head or tail of a dotted name, or when to do the "finish