python/google/appengine/tools/dev_appserver_import_hook.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20 """Import hook for dev_appserver.py."""
  21
  22 import dummy_thread
  23 import errno
  24 import imp
  25 import inspect
  26 import itertools
  27 import locale
  28 import logging
  29 import mimetypes
  30 import os
  31 import pickle
  32 import random
  33 import re
  34 import sys
  35 import urllib
  36
  37 try:
  38   import distutils.util
  39 except ImportError:
  40
  41
  42
  43   pass
  44
  45
  46
  47
  48 from google.appengine import dist
  49 from google.appengine import dist27 as dist27
  50
  51 from google.appengine.api import appinfo
  52
  53
  54 SITE_PACKAGES = os.path.normcase(os.path.join(os.path.dirname(os.__file__),
  55                                               'site-packages'))
  56
  57
  58 import google.appengine
  59 SDK_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
  60     google.appengine.__file__)))
  61
  62
  63 CODING_COOKIE_RE = re.compile("coding[:=]\s*([-\w.]+)")
  64 DEFAULT_ENCODING = 'ascii'
  65
  66
  67 def FakeURandom(n):
  68   """Fake version of os.urandom."""
  69   bytes = ''
  70   for _ in range(n):
  71     bytes += chr(random.randint(0, 255))
  72   return bytes
  73
  74
  75 def FakeUname():
  76   """Fake version of os.uname."""
  77   return ('Linux', '', '', '', '')
  78
  79
  80 def FakeUnlink(path):
  81   """Fake version of os.unlink."""
  82   if os.path.isdir(path):
  83     raise OSError(errno.ENOENT, "Is a directory", path)
  84   else:
  85     raise OSError(errno.EPERM, "Operation not permitted", path)
  86
  87
  88 def FakeReadlink(path):
  89   """Fake version of os.readlink."""
  90   raise OSError(errno.EINVAL, "Invalid argument", path)
  91
  92
  93 def FakeAccess(path, mode):
  94   """Fake version of os.access where only reads are supported."""
  95   if not os.path.exists(path) or mode != os.R_OK:
  96     return False
  97   else:
  98     return True
  99
 100
 101 def FakeSetLocale(category, value=None, original_setlocale=locale.setlocale):
 102   """Fake version of locale.setlocale that only supports the default."""
 103   if value not in (None, '', 'C', 'POSIX'):
 104     raise locale.Error('locale emulation only supports "C" locale')
 105   return original_setlocale(category, 'C')
 106
 107
 108 def FakeOpen(filename, flags, mode=0777):
 109   """Fake version of os.open."""
 110   raise OSError(errno.EPERM, "Operation not permitted", filename)
 111
 112
 113 def FakeRename(src, dst):
 114   """Fake version of os.rename."""
 115   raise OSError(errno.EPERM, "Operation not permitted", src)
 116
 117
 118 def FakeUTime(path, times):
 119   """Fake version of os.utime."""
 120   raise OSError(errno.EPERM, "Operation not permitted", path)
 121
 122
 123 def FakeGetPlatform():
 124   """Fake distutils.util.get_platform on OS/X.  Pass-through otherwise."""
 125   if sys.platform == 'darwin':
 126     return 'macosx-'
 127   else:
 128     return distutils.util.get_platform()
 129
 130
 131 def FakeCryptoRandomOSRNGnew(*args, **kwargs):
 132   from Crypto.Random.OSRNG import fallback
 133   return fallback.new(*args, **kwargs)
 134
 135
 136
 137
 138
 139
 140 def NeedsMacOSXProxyFakes():
 141   """Returns True if the MacOS X urllib fakes should be installed."""
 142   return (sys.platform == 'darwin' and
 143           (2, 6, 0) <= sys.version_info < (2, 6, 4))
 144
 145
 146 if NeedsMacOSXProxyFakes():
 147   def _FakeProxyBypassHelper(fn,
 148                              original_module_dict=sys.modules.copy(),
 149                              original_uname=os.uname):
 150     """Setups and restores the state for the Mac OS X urllib fakes."""
 151     def Inner(*args, **kwargs):
 152       current_uname = os.uname
 153       current_meta_path = sys.meta_path[:]
 154       current_modules = sys.modules.copy()
 155
 156       try:
 157         sys.modules.clear()
 158         sys.modules.update(original_module_dict)
 159         sys.meta_path[:] = []
 160         os.uname = original_uname
 161
 162         return fn(*args, **kwargs)
 163       finally:
 164         sys.modules.clear()
 165         sys.modules.update(current_modules)
 166         os.uname = current_uname
 167         sys.meta_path[:] = current_meta_path
 168     return Inner
 169
 170
 171   @_FakeProxyBypassHelper
 172   def FakeProxyBypassMacOSXSysconf(
 173       host,
 174       original_proxy_bypass_macosx_sysconf=urllib.proxy_bypass_macosx_sysconf):
 175     """Fake for urllib.proxy_bypass_macosx_sysconf for Python 2.6.0 to 2.6.3."""
 176     return original_proxy_bypass_macosx_sysconf(host)
 177
 178
 179   @_FakeProxyBypassHelper
 180   def FakeGetProxiesMacOSXSysconf(
 181       original_getproxies_macosx_sysconf=urllib.getproxies_macosx_sysconf):
 182     """Fake for urllib.getproxies_macosx_sysconf for Python 2.6.0 to 2.6.3."""
 183     return original_getproxies_macosx_sysconf()
 184
 185
 186 def IsPathInSubdirectories(filename,
 187                            subdirectories,
 188                            normcase=os.path.normcase):
 189   """Determines if a filename is contained within one of a set of directories.
 190
 191   Args:
 192     filename: Path of the file (relative or absolute).
 193     subdirectories: Iterable collection of paths to subdirectories which the
 194       given filename may be under.
 195     normcase: Used for dependency injection.
 196
 197   Returns:
 198     True if the supplied filename is in one of the given sub-directories or
 199     its hierarchy of children. False otherwise.
 200   """
 201   file_dir = normcase(os.path.dirname(os.path.abspath(filename)))
 202   for parent in subdirectories:
 203     fixed_parent = normcase(os.path.abspath(parent))
 204     if os.path.commonprefix([file_dir, fixed_parent]) == fixed_parent:
 205       return True
 206   return False
 207
 208
 209 def GeneratePythonPaths(*p):
 210   """Generate all valid filenames for the given file.
 211
 212   Args:
 213     p: Positional args are the folders to the file and finally the file
 214        without a suffix.
 215
 216   Returns:
 217     A list of strings representing the given path to a file with each valid
 218       suffix for this python build.
 219   """
 220   suffixes = imp.get_suffixes()
 221   return [os.path.join(*p) + s for s, m, t in suffixes]
 222
 223
 224 class FakeFile(file):
 225   """File sub-class that enforces the security restrictions of the production
 226   environment.
 227   """
 228
 229   ALLOWED_MODES = frozenset(['r', 'rb', 'U', 'rU'])
 230
 231
 232   ALLOWED_FILES = set(os.path.normcase(filename)
 233                       for filename in mimetypes.knownfiles
 234                       if os.path.isfile(filename))
 235
 236
 237   ALLOWED_FILES_RE = set([re.compile(r'.*/python27.zip$')])
 238
 239
 240
 241
 242
 243
 244   ALLOWED_DIRS = set([
 245       os.path.normcase(os.path.realpath(os.path.dirname(os.__file__))),
 246       os.path.normcase(os.path.abspath(os.path.dirname(os.__file__))),
 247       os.path.normcase(os.path.dirname(os.path.realpath(os.__file__))),
 248       os.path.normcase(os.path.dirname(os.path.abspath(os.__file__))),
 249   ])
 250   os_source_location = inspect.getsourcefile(os)
 251
 252   if os_source_location is not None:
 253
 254
 255
 256     ALLOWED_DIRS.update([
 257         os.path.normcase(os.path.realpath(os.path.dirname(os_source_location))),
 258         os.path.normcase(os.path.abspath(os.path.dirname(os_source_location))),
 259         os.path.normcase(os.path.dirname(os.path.realpath(os_source_location))),
 260         os.path.normcase(os.path.dirname(os.path.abspath(os_source_location))),
 261     ])
 262
 263
 264
 265
 266   NOT_ALLOWED_DIRS = set([
 267
 268
 269
 270
 271       SITE_PACKAGES,
 272   ])
 273
 274
 275
 276
 277
 278
 279
 280
 281   ALLOWED_SITE_PACKAGE_DIRS = set(
 282       os.path.normcase(os.path.abspath(os.path.join(SITE_PACKAGES, path)))
 283       for path in [
 284
 285           ])
 286
 287   ALLOWED_SITE_PACKAGE_FILES = set(
 288       os.path.normcase(os.path.abspath(os.path.join(
 289           os.path.dirname(os.__file__), 'site-packages', path)))
 290       for path in itertools.chain(*[
 291
 292           [os.path.join('Crypto')],
 293           GeneratePythonPaths('Crypto', '__init__'),
 294           GeneratePythonPaths('Crypto', 'pct_warnings'),
 295           [os.path.join('Crypto', 'Cipher')],
 296           GeneratePythonPaths('Crypto', 'Cipher', '__init__'),
 297           GeneratePythonPaths('Crypto', 'Cipher', 'blockalgo'),
 298           GeneratePythonPaths('Crypto', 'Cipher', 'AES'),
 299           GeneratePythonPaths('Crypto', 'Cipher', 'ARC2'),
 300           GeneratePythonPaths('Crypto', 'Cipher', 'ARC4'),
 301           GeneratePythonPaths('Crypto', 'Cipher', 'Blowfish'),
 302           GeneratePythonPaths('Crypto', 'Cipher', 'CAST'),
 303           GeneratePythonPaths('Crypto', 'Cipher', 'DES'),
 304           GeneratePythonPaths('Crypto', 'Cipher', 'DES3'),
 305           GeneratePythonPaths('Crypto', 'Cipher', 'PKCS1_OAEP'),
 306           GeneratePythonPaths('Crypto', 'Cipher', 'PKCS1_v1_5'),
 307           GeneratePythonPaths('Crypto', 'Cipher', 'XOR'),
 308           [os.path.join('Crypto', 'Hash')],
 309           GeneratePythonPaths('Crypto', 'Hash', '__init__'),
 310           GeneratePythonPaths('Crypto', 'Hash', 'hashalgo'),
 311           GeneratePythonPaths('Crypto', 'Hash', 'HMAC'),
 312           GeneratePythonPaths('Crypto', 'Hash', 'MD2'),
 313           GeneratePythonPaths('Crypto', 'Hash', 'MD4'),
 314           GeneratePythonPaths('Crypto', 'Hash', 'MD5'),
 315           GeneratePythonPaths('Crypto', 'Hash', 'SHA'),
 316           GeneratePythonPaths('Crypto', 'Hash', 'SHA224'),
 317           GeneratePythonPaths('Crypto', 'Hash', 'SHA256'),
 318           GeneratePythonPaths('Crypto', 'Hash', 'SHA384'),
 319           GeneratePythonPaths('Crypto', 'Hash', 'SHA512'),
 320           GeneratePythonPaths('Crypto', 'Hash', 'RIPEMD'),
 321           [os.path.join('Crypto', 'Protocol')],
 322           GeneratePythonPaths('Crypto', 'Protocol', '__init__'),
 323           GeneratePythonPaths('Crypto', 'Protocol', 'AllOrNothing'),
 324           GeneratePythonPaths('Crypto', 'Protocol', 'Chaffing'),
 325           GeneratePythonPaths('Crypto', 'Protocol', 'KDF'),
 326           [os.path.join('Crypto', 'PublicKey')],
 327           GeneratePythonPaths('Crypto', 'PublicKey', '__init__'),
 328           GeneratePythonPaths('Crypto', 'PublicKey', 'DSA'),
 329           GeneratePythonPaths('Crypto', 'PublicKey', '_DSA'),
 330           GeneratePythonPaths('Crypto', 'PublicKey', 'ElGamal'),
 331           GeneratePythonPaths('Crypto', 'PublicKey', 'RSA'),
 332           GeneratePythonPaths('Crypto', 'PublicKey', '_RSA'),
 333           GeneratePythonPaths('Crypto', 'PublicKey', 'pubkey'),
 334           GeneratePythonPaths('Crypto', 'PublicKey', 'qNEW'),
 335           GeneratePythonPaths('Crypto', 'PublicKey', '_slowmath'),
 336           [os.path.join('Crypto', 'Random')],
 337           GeneratePythonPaths('Crypto', 'Random', '__init__'),
 338           GeneratePythonPaths('Crypto', 'Random', 'random'),
 339           GeneratePythonPaths('Crypto', 'Random', '_UserFriendlyRNG'),
 340           [os.path.join('Crypto', 'Random', 'OSRNG')],
 341           GeneratePythonPaths('Crypto', 'Random', 'OSRNG', '__init__'),
 342           GeneratePythonPaths('Crypto', 'Random', 'OSRNG', 'fallback'),
 343           GeneratePythonPaths('Crypto', 'Random', 'OSRNG', 'nt'),
 344           GeneratePythonPaths('Crypto', 'Random', 'OSRNG', 'posix'),
 345           GeneratePythonPaths('Crypto', 'Random', 'OSRNG', 'rng_base'),
 346           [os.path.join('Crypto', 'Random', 'Fortuna')],
 347           GeneratePythonPaths('Crypto', 'Random', 'Fortuna', '__init__'),
 348           GeneratePythonPaths('Crypto', 'Random', 'Fortuna',
 349                               'FortunaAccumulator'),
 350           GeneratePythonPaths('Crypto', 'Random', 'Fortuna',
 351                               'FortunaGenerator'),
 352           GeneratePythonPaths('Crypto', 'Random', 'Fortuna', 'SHAd256'),
 353           [os.path.join('Crypto', 'Signature')],
 354           GeneratePythonPaths('Crypto', 'Signature', '__init__'),
 355           GeneratePythonPaths('Crypto', 'Signature', 'PKCS1_PSS'),
 356           GeneratePythonPaths('Crypto', 'Signature', 'PKCS1_v1_5'),
 357           [os.path.join('Crypto', 'Util')],
 358           GeneratePythonPaths('Crypto', 'Util', '__init__'),
 359           GeneratePythonPaths('Crypto', 'Util', 'asn1'),
 360           GeneratePythonPaths('Crypto', 'Util', 'Counter'),
 361           GeneratePythonPaths('Crypto', 'Util', 'RFC1751'),
 362           GeneratePythonPaths('Crypto', 'Util', 'number'),
 363           GeneratePythonPaths('Crypto', 'Util', '_number_new'),
 364           GeneratePythonPaths('Crypto', 'Util', 'py3compat'),
 365           GeneratePythonPaths('Crypto', 'Util', 'python_compat'),
 366           GeneratePythonPaths('Crypto', 'Util', 'randpool'),
 367           ]))
 368
 369
 370
 371   _original_file = file
 372
 373
 374   _root_path = None
 375   _application_paths = None
 376   _skip_files = None
 377   _static_file_config_matcher = None
 378
 379
 380   _allow_skipped_files = True
 381
 382
 383   _availability_cache = {}
 384
 385   @staticmethod
 386   def SetAllowedPaths(root_path, application_paths):
 387     """Configures which paths are allowed to be accessed.
 388
 389     Must be called at least once before any file objects are created in the
 390     hardened environment.
 391
 392     Args:
 393       root_path: Absolute path to the root of the application.
 394       application_paths: List of additional paths that the application may
 395                          access, this must include the App Engine runtime but
 396                          not the Python library directories.
 397     """
 398
 399
 400     FakeFile._application_paths = (set(os.path.realpath(path)
 401                                        for path in application_paths) |
 402                                    set(os.path.abspath(path)
 403                                        for path in application_paths))
 404     FakeFile._application_paths.add(root_path)
 405
 406
 407     FakeFile._root_path = os.path.join(root_path, '')
 408
 409     FakeFile._availability_cache = {}
 410
 411   @staticmethod
 412   def SetAllowSkippedFiles(allow_skipped_files):
 413     """Configures access to files matching FakeFile._skip_files.
 414
 415     Args:
 416       allow_skipped_files: Boolean whether to allow access to skipped files
 417     """
 418     FakeFile._allow_skipped_files = allow_skipped_files
 419     FakeFile._availability_cache = {}
 420
 421   @staticmethod
 422   def SetAllowedModule(name):
 423     """Allow the use of a module based on where it is located.
 424
 425     Meant to be used by use_library() so that it has a link back into the
 426     trusted part of the interpreter.
 427
 428     Args:
 429       name: Name of the module to allow.
 430     """
 431     stream, pathname, description = imp.find_module(name)
 432     pathname = os.path.normcase(os.path.abspath(pathname))
 433     if stream:
 434       stream.close()
 435       FakeFile.ALLOWED_FILES.add(pathname)
 436       FakeFile.ALLOWED_FILES.add(os.path.realpath(pathname))
 437     else:
 438       assert description[2] == imp.PKG_DIRECTORY
 439       if pathname.startswith(SITE_PACKAGES):
 440         FakeFile.ALLOWED_SITE_PACKAGE_DIRS.add(pathname)
 441         FakeFile.ALLOWED_SITE_PACKAGE_DIRS.add(os.path.realpath(pathname))
 442       else:
 443         FakeFile.ALLOWED_DIRS.add(pathname)
 444         FakeFile.ALLOWED_DIRS.add(os.path.realpath(pathname))
 445
 446   @staticmethod
 447   def SetSkippedFiles(skip_files):
 448     """Sets which files in the application directory are to be ignored.
 449
 450     Must be called at least once before any file objects are created in the
 451     hardened environment.
 452
 453     Must be called whenever the configuration was updated.
 454
 455     Args:
 456       skip_files: Object with .match() method (e.g. compiled regexp).
 457     """
 458     FakeFile._skip_files = skip_files
 459     FakeFile._availability_cache = {}
 460
 461   @staticmethod
 462   def SetStaticFileConfigMatcher(static_file_config_matcher):
 463     """Sets StaticFileConfigMatcher instance for checking if a file is static.
 464
 465     Must be called at least once before any file objects are created in the
 466     hardened environment.
 467
 468     Must be called whenever the configuration was updated.
 469
 470     Args:
 471       static_file_config_matcher: StaticFileConfigMatcher instance.
 472     """
 473     FakeFile._static_file_config_matcher = static_file_config_matcher
 474     FakeFile._availability_cache = {}
 475
 476   @staticmethod
 477   def IsFileAccessible(filename, normcase=os.path.normcase,
 478                        py27_optional=False):
 479     """Determines if a file's path is accessible.
 480
 481     SetAllowedPaths(), SetSkippedFiles() and SetStaticFileConfigMatcher() must
 482     be called before this method or else all file accesses will raise an error.
 483
 484     Args:
 485       filename: Path of the file to check (relative or absolute). May be a
 486         directory, in which case access for files inside that directory will
 487         be checked.
 488       normcase: Used for dependency injection.
 489       py27_optional: Whether the filename being checked matches the name of an
 490         optional python27 runtime library.
 491
 492     Returns:
 493       True if the file is accessible, False otherwise.
 494     """
 495
 496
 497
 498     logical_filename = normcase(os.path.abspath(filename))
 499
 500
 501
 502
 503
 504
 505
 506     result = FakeFile._availability_cache.get(logical_filename)
 507     if result is None:
 508       result = FakeFile._IsFileAccessibleNoCache(logical_filename,
 509                                                  normcase=normcase,
 510                                                  py27_optional=py27_optional)
 511       FakeFile._availability_cache[logical_filename] = result
 512     return result
 513
 514   @staticmethod
 515   def _IsFileAccessibleNoCache(logical_filename, normcase=os.path.normcase,
 516                                py27_optional=False):
 517     """Determines if a file's path is accessible.
 518
 519     This is an internal part of the IsFileAccessible implementation.
 520
 521     Args:
 522       logical_filename: Absolute path of the file to check.
 523       normcase: Used for dependency injection.
 524       py27_optional: Whether the filename being checked matches the name of an
 525         optional python27 runtime library.
 526
 527     Returns:
 528       True if the file is accessible, False otherwise.
 529     """
 530
 531
 532
 533
 534     logical_dirfakefile = logical_filename
 535     is_dir = False
 536     if os.path.isdir(logical_filename):
 537       logical_dirfakefile = os.path.join(logical_filename, 'foo')
 538       is_dir = True
 539
 540
 541     if IsPathInSubdirectories(logical_dirfakefile, [FakeFile._root_path],
 542                               normcase=normcase):
 543
 544       relative_filename = logical_dirfakefile[len(FakeFile._root_path):]
 545
 546       if not FakeFile._allow_skipped_files:
 547         path = relative_filename
 548         if is_dir:
 549
 550
 551
 552
 553           path = os.path.dirname(path)
 554         while path != os.path.dirname(path):
 555           if FakeFile._skip_files.match(path):
 556             logging.warning('Blocking access to skipped file "%s"',
 557                             logical_filename)
 558             return False
 559           path = os.path.dirname(path)
 560
 561       if FakeFile._static_file_config_matcher.IsStaticFile(relative_filename):
 562         logging.warning('Blocking access to static file "%s"',
 563                         logical_filename)
 564         return False
 565
 566     if py27_optional:
 567
 568
 569       return True
 570
 571     if logical_filename in FakeFile.ALLOWED_FILES:
 572       return True
 573
 574     for regex in FakeFile.ALLOWED_FILES_RE:
 575       match = regex.match(logical_filename)
 576       if match and match.end() == len(logical_filename):
 577         return True
 578
 579     if logical_filename in FakeFile.ALLOWED_SITE_PACKAGE_FILES:
 580       return True
 581
 582     if IsPathInSubdirectories(logical_dirfakefile,
 583                               FakeFile.ALLOWED_SITE_PACKAGE_DIRS,
 584                               normcase=normcase):
 585       return True
 586
 587     allowed_dirs = FakeFile._application_paths | FakeFile.ALLOWED_DIRS
 588     if (IsPathInSubdirectories(logical_dirfakefile,
 589                                allowed_dirs,
 590                                normcase=normcase) and
 591         not IsPathInSubdirectories(logical_dirfakefile,
 592                                    FakeFile.NOT_ALLOWED_DIRS,
 593                                    normcase=normcase)):
 594       return True
 595
 596     return False
 597
 598   def __init__(self, filename, mode='r', bufsize=-1, **kwargs):
 599     """Initializer. See file built-in documentation."""
 600     if mode not in FakeFile.ALLOWED_MODES:
 601
 602
 603
 604
 605       raise IOError('invalid mode: %s' % mode)
 606
 607     if not FakeFile.IsFileAccessible(filename):
 608       raise IOError(errno.EACCES, 'file not accessible', filename)
 609
 610     super(FakeFile, self).__init__(filename, mode, bufsize, **kwargs)
 611
 612
 613
 614 dist._library.SetAllowedModule = FakeFile.SetAllowedModule
 615
 616
 617 class RestrictedPathFunction(object):
 618   """Enforces access restrictions for functions that have a file or
 619   directory path as their first argument."""
 620
 621   _original_os = os
 622
 623   def __init__(self, original_func):
 624     """Initializer.
 625
 626     Args:
 627       original_func: Callable that takes as its first argument the path to a
 628         file or directory on disk; all subsequent arguments may be variable.
 629     """
 630     self._original_func = original_func
 631
 632   def __call__(self, path, *args, **kwargs):
 633     """Enforces access permissions for the function passed to the constructor.
 634     """
 635     if not FakeFile.IsFileAccessible(path):
 636       raise OSError(errno.EACCES, 'path not accessible', path)
 637
 638     return self._original_func(path, *args, **kwargs)
 639
 640
 641 def GetSubmoduleName(fullname):
 642   """Determines the leaf submodule name of a full module name.
 643
 644   Args:
 645     fullname: Fully qualified module name, e.g. 'foo.bar.baz'
 646
 647   Returns:
 648     Submodule name, e.g. 'baz'. If the supplied module has no submodule (e.g.,
 649     'stuff'), the returned value will just be that module name ('stuff').
 650   """
 651   return fullname.rsplit('.', 1)[-1]
 652
 653
 654 class CouldNotFindModuleError(ImportError):
 655   """Raised when a module could not be found.
 656
 657   In contrast to when a module has been found, but cannot be loaded because of
 658   hardening restrictions.
 659   """
 660
 661
 662 class Py27OptionalModuleError(ImportError):
 663   """Raised for error conditions relating to optional Python 2.7 modules."""
 664
 665
 666 def Trace(func):
 667   """Call stack logging decorator for HardenedModulesHook class.
 668
 669   This decorator logs the call stack of the HardenedModulesHook class as
 670   it executes, indenting logging messages based on the current stack depth.
 671
 672   Args:
 673     func: the function to decorate.
 674
 675   Returns:
 676     The decorated function.
 677   """
 678
 679   def Decorate(self, *args, **kwargs):
 680     args_to_show = []
 681     if args is not None:
 682       args_to_show.extend(str(argument) for argument in args)
 683     if kwargs is not None:
 684       args_to_show.extend('%s=%s' % (key, value)
 685                           for key, value in kwargs.iteritems())
 686
 687     args_string = ', '.join(args_to_show)
 688
 689     self.log('Entering %s(%s)', func.func_name, args_string)
 690     self._indent_level += 1
 691     try:
 692       return func(self, *args, **kwargs)
 693     finally:
 694       self._indent_level -= 1
 695       self.log('Exiting %s(%s)', func.func_name, args_string)
 696
 697   return Decorate
 698
 699
 700 class HardenedModulesHook(object):
 701   """Meta import hook that restricts the modules used by applications to match
 702   the production environment.
 703
 704   Module controls supported:
 705   - Disallow native/extension modules from being loaded
 706   - Disallow built-in and/or Python-distributed modules from being loaded
 707   - Replace modules with completely empty modules
 708   - Override specific module attributes
 709   - Replace one module with another
 710
 711   After creation, this object should be added to the front of the sys.meta_path
 712   list (which may need to be created). The sys.path_importer_cache dictionary
 713   should also be cleared, to prevent loading any non-restricted modules.
 714
 715   See PEP302 for more info on how this works:
 716     http://www.python.org/dev/peps/pep-0302/
 717   """
 718
 719
 720
 721   ENABLE_LOGGING = False
 722
 723   def log(self, message, *args):
 724     """Logs an import-related message to stderr, with indentation based on
 725     current call-stack depth.
 726
 727     Args:
 728       message: Logging format string.
 729       args: Positional format parameters for the logging message.
 730     """
 731     if HardenedModulesHook.ENABLE_LOGGING:
 732       indent = self._indent_level * '  '
 733       print >>sys.__stderr__, indent + (message % args)
 734
 735
 736
 737
 738
 739
 740   _WHITE_LIST_C_MODULES = [
 741       'py_streamhtmlparser',
 742       'AES',
 743       'ARC2',
 744       'ARC4',
 745       'Blowfish',
 746       'CAST',
 747       'DES',
 748       'DES3',
 749       'MD2',
 750       'MD4',
 751       'RIPEMD',
 752       'RIPEMD160',
 753       'SHA256',
 754       'XOR',
 755
 756       '_AES',
 757       '_ARC2',
 758       '_ARC4',
 759       '_Blowfish',
 760       '_CAST',
 761       '_DES',
 762       '_DES3',
 763       '_MD2',
 764       '_MD4',
 765       '_RIPEMD160',
 766       '_SHA224',
 767       '_SHA256',
 768       '_SHA384',
 769       '_SHA512',
 770       '_XOR',
 771
 772       '_Crypto_Cipher__AES',
 773       '_Crypto_Cipher__ARC2',
 774       '_Crypto_Cipher__ARC4',
 775       '_Crypto_Cipher__Blowfish',
 776       '_Crypto_Cipher__CAST',
 777       '_Crypto_Cipher__DES',
 778       '_Crypto_Cipher__DES3',
 779       '_Crypto_Cipher__XOR',
 780       '_Crypto_Hash__MD2',
 781       '_Crypto_Hash__MD4',
 782       '_Crypto_Hash__RIPEMD',
 783       '_Crypto_Hash__SHA256',
 784       'array',
 785       'binascii',
 786       'bz2',
 787       'cmath',
 788       'collections',
 789       'crypt',
 790       'cStringIO',
 791       'datetime',
 792       'errno',
 793       'exceptions',
 794       'gc',
 795       'itertools',
 796       'math',
 797       'md5',
 798       'operator',
 799       'posix',
 800       'posixpath',
 801       'pyexpat',
 802       'sha',
 803       'struct',
 804       'strxor',
 805       'sys',
 806       'time',
 807       'timing',
 808       'unicodedata',
 809       'zlib',
 810       '_ast',
 811       '_bisect',
 812       '_codecs',
 813       '_codecs_cn',
 814       '_codecs_hk',
 815       '_codecs_iso2022',
 816       '_codecs_jp',
 817       '_codecs_kr',
 818       '_codecs_tw',
 819       '_collections',
 820       '_counter',
 821       '_csv',
 822       '_elementtree',
 823       '_fastmath',
 824       '_functools',
 825       '_hashlib',
 826       '_heapq',
 827       '_io',
 828       '_locale',
 829       '_lsprof',
 830       '_md5',
 831       '_multibytecodec',
 832       '_scproxy',
 833       '_random',
 834       '_sha',
 835       '_sha256',
 836       '_sha512',
 837       '_sre',
 838       '_struct',
 839       '_types',
 840       '_weakref',
 841       '__main__',
 842   ]
 843
 844
 845
 846
 847   _PY27_ALLOWED_MODULES = [
 848     '_bytesio',
 849     '_fileio',
 850     '_json',
 851     '_symtable',
 852     '_yaml',
 853     'parser',
 854     'strop',
 855
 856
 857
 858
 859
 860
 861   ]
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878   __PY27_OPTIONAL_ALLOWED_MODULES = {
 879
 880     'django': [],
 881     'endpoints': [],
 882     'jinja2': ['_debugsupport', '_speedups'],
 883     'lxml': ['etree', 'objectify'],
 884     'markupsafe': ['_speedups'],
 885     'matplotlib': [
 886       'ft2font',
 887       'ttconv',
 888       '_png',
 889       '_backend_agg',
 890       '_path',
 891       '_image',
 892       '_cntr',
 893       'nxutils',
 894       '_delaunay',
 895       '_tri',
 896     ],
 897     'numpy': [
 898       '_capi',
 899       '_compiled_base',
 900       '_dotblas',
 901       'fftpack_lite',
 902       'lapack_lite',
 903       'mtrand',
 904       'multiarray',
 905       'scalarmath',
 906       '_sort',
 907       'umath',
 908       'umath_tests',
 909     ],
 910     'PIL': ['_imaging', '_imagingcms', '_imagingft', '_imagingmath'],
 911
 912
 913
 914     'setuptools': [],
 915
 916
 917   }
 918
 919   __CRYPTO_CIPHER_ALLOWED_MODULES = [
 920       'MODE_CBC',
 921       'MODE_CFB',
 922       'MODE_CTR',
 923       'MODE_ECB',
 924       'MODE_OFB',
 925       'block_size',
 926       'key_size',
 927       'new',
 928   ]
 929
 930
 931
 932
 933   _WHITE_LIST_PARTIAL_MODULES = {
 934       'Crypto.Cipher.AES': __CRYPTO_CIPHER_ALLOWED_MODULES,
 935       'Crypto.Cipher.ARC2': __CRYPTO_CIPHER_ALLOWED_MODULES,
 936       'Crypto.Cipher.Blowfish': __CRYPTO_CIPHER_ALLOWED_MODULES,
 937       'Crypto.Cipher.CAST': __CRYPTO_CIPHER_ALLOWED_MODULES,
 938       'Crypto.Cipher.DES': __CRYPTO_CIPHER_ALLOWED_MODULES,
 939       'Crypto.Cipher.DES3': __CRYPTO_CIPHER_ALLOWED_MODULES,
 940
 941
 942       'gc': [
 943           'enable',
 944           'disable',
 945           'isenabled',
 946           'collect',
 947           'get_debug',
 948           'set_threshold',
 949           'get_threshold',
 950           'get_count'
 951       ],
 952
 953
 954
 955
 956       'os': [
 957           'access',
 958           'altsep',
 959           'curdir',
 960           'defpath',
 961           'devnull',
 962           'environ',
 963           'error',
 964           'extsep',
 965           'EX_NOHOST',
 966           'EX_NOINPUT',
 967           'EX_NOPERM',
 968           'EX_NOUSER',
 969           'EX_OK',
 970           'EX_OSERR',
 971           'EX_OSFILE',
 972           'EX_PROTOCOL',
 973           'EX_SOFTWARE',
 974           'EX_TEMPFAIL',
 975           'EX_UNAVAILABLE',
 976           'EX_USAGE',
 977           'F_OK',
 978           'getcwd',
 979           'getcwdu',
 980           'getenv',
 981
 982           'listdir',
 983           'lstat',
 984           'name',
 985           'NGROUPS_MAX',
 986           'O_APPEND',
 987           'O_CREAT',
 988           'O_DIRECT',
 989           'O_DIRECTORY',
 990           'O_DSYNC',
 991           'O_EXCL',
 992           'O_LARGEFILE',
 993           'O_NDELAY',
 994           'O_NOCTTY',
 995           'O_NOFOLLOW',
 996           'O_NONBLOCK',
 997           'O_RDONLY',
 998           'O_RDWR',
 999           'O_RSYNC',
1000           'O_SYNC',
1001           'O_TRUNC',
1002           'O_WRONLY',
1003           'open',
1004           'pardir',
1005           'path',
1006           'pathsep',
1007           'R_OK',
1008           'readlink',
1009           'remove',
1010           'rename',
1011           'SEEK_CUR',
1012           'SEEK_END',
1013           'SEEK_SET',
1014           'sep',
1015           'stat',
1016           'stat_float_times',
1017           'stat_result',
1018           'strerror',
1019           'TMP_MAX',
1020           'unlink',
1021           'urandom',
1022           'utime',
1023           'walk',
1024           'WCOREDUMP',
1025           'WEXITSTATUS',
1026           'WIFEXITED',
1027           'WIFSIGNALED',
1028           'WIFSTOPPED',
1029           'WNOHANG',
1030           'WSTOPSIG',
1031           'WTERMSIG',
1032           'WUNTRACED',
1033           'W_OK',
1034           'X_OK',
1035           '_get_exports_list',
1036       ],
1037
1038
1039       'signal': [
1040       ],
1041
1042       'ssl': [
1043       ],
1044   }
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054   _MODULE_OVERRIDES = {
1055       'locale': {
1056           'setlocale': FakeSetLocale,
1057       },
1058
1059       'os': {
1060           'access': FakeAccess,
1061           'listdir': RestrictedPathFunction(os.listdir),
1062
1063           'lstat': RestrictedPathFunction(os.stat),
1064           'open': FakeOpen,
1065           'readlink': FakeReadlink,
1066           'remove': FakeUnlink,
1067           'rename': FakeRename,
1068           'stat': RestrictedPathFunction(os.stat),
1069           'uname': FakeUname,
1070           'unlink': FakeUnlink,
1071           'urandom': FakeURandom,
1072           'utime': FakeUTime,
1073       },
1074
1075       'signal': {
1076
1077           '__doc__': None,
1078       },
1079
1080       'distutils.util': {
1081           'get_platform': FakeGetPlatform,
1082       },
1083
1084       'Crypto.Random.OSRNG': {
1085           'new': FakeCryptoRandomOSRNGnew,
1086       },
1087   }
1088
1089
1090   _ENABLED_FILE_TYPES = (
1091       imp.PKG_DIRECTORY,
1092       imp.PY_SOURCE,
1093       imp.PY_COMPILED,
1094       imp.C_BUILTIN,
1095   )
1096
1097   def __init__(self,
1098                config,
1099                module_dict,
1100                app_code_path,
1101                imp_module=imp,
1102                os_module=os,
1103                dummy_thread_module=dummy_thread,
1104                pickle_module=pickle):
1105     """Initializer.
1106
1107     Args:
1108       config: AppInfoExternal instance representing the parsed app.yaml file.
1109       module_dict: Module dictionary to use for managing system modules.
1110         Should be sys.modules.
1111       app_code_path: The absolute path to the application code on disk.
1112       imp_module, os_module, dummy_thread_module, etc.: References to
1113         modules that exist in the dev_appserver that must be used by this class
1114         in order to function, even if these modules have been unloaded from
1115         sys.modules.
1116     """
1117     self._config = config
1118     self._module_dict = module_dict
1119     self._imp = imp_module
1120     self._os = os_module
1121     self._dummy_thread = dummy_thread_module
1122     self._pickle = pickle
1123     self._indent_level = 0
1124     self._app_code_path = app_code_path
1125     self._white_list_c_modules = list(self._WHITE_LIST_C_MODULES)
1126     self._white_list_partial_modules = dict(self._WHITE_LIST_PARTIAL_MODULES)
1127     self._enabled_modules = []
1128
1129     if self._config and self._config.runtime == 'python27':
1130       self._white_list_c_modules.extend(self._PY27_ALLOWED_MODULES)
1131
1132
1133
1134
1135       self._white_list_partial_modules['os'] = (
1136         list(self._white_list_partial_modules['os']) +
1137         ['getpid', 'getuid', 'sys'])
1138
1139
1140       for k in self._white_list_partial_modules.keys():
1141         if k.startswith('Crypto'):
1142           del self._white_list_partial_modules[k]
1143
1144
1145       webob_path = os.path.join(SDK_ROOT, 'lib', 'webob-1.1.1')
1146       if webob_path not in sys.path:
1147         sys.path.insert(1, webob_path)
1148
1149       for libentry in self._config.GetAllLibraries():
1150         self._enabled_modules.append(libentry.name)
1151         extra = self.__PY27_OPTIONAL_ALLOWED_MODULES.get(libentry.name)
1152         logging.debug('Enabling %s: %r', libentry.name, extra)
1153         if extra:
1154           self._white_list_c_modules.extend(extra)
1155         if libentry.name == 'django':
1156
1157
1158
1159           if 'django' not in self._module_dict:
1160             version = libentry.version
1161             if version == 'latest':
1162               django_library = appinfo._NAME_TO_SUPPORTED_LIBRARY['django']
1163               version = django_library.non_deprecated_versions[-1]
1164             if google.__name__.endswith('3'):
1165
1166
1167               try:
1168                 __import__('django.v' + version.replace('.', '_'))
1169                 continue
1170               except ImportError:
1171                 sys.modules.pop('django', None)
1172             sitedir = os.path.join(SDK_ROOT,
1173                                    'lib',
1174                                    'django-%s' % version)
1175             if os.path.isdir(sitedir):
1176               logging.debug('Enabling Django version %s at %s',
1177                             version, sitedir)
1178               sys.path[:] = [dirname
1179                              for dirname in sys.path
1180                              if not dirname.startswith(os.path.join(
1181                                SDK_ROOT, 'lib', 'django'))]
1182               sys.path.insert(1, sitedir)
1183             else:
1184               logging.warn('Enabling Django version %s (no directory found)',
1185                            version)
1186         elif libentry.name == 'endpoints':
1187           try:
1188
1189             from google.third_party.apphosting.python.endpoints import v1_0
1190             sys.path.append(os.path.dirname(v1_0.__file__))
1191             del v1_0
1192           except ImportError:
1193
1194
1195             endpoints_path = os.path.join(SDK_ROOT, 'lib', 'endpoints-1.0')
1196             if endpoints_path not in sys.path:
1197               sys.path.append(endpoints_path)
1198
1199
1200   @Trace
1201   def find_module(self, fullname, path=None):
1202     """See PEP 302."""
1203
1204
1205
1206
1207     if fullname in ('cPickle', 'thread'):
1208       return self
1209
1210     search_path = path
1211     all_modules = fullname.split('.')
1212     try:
1213       for index, current_module in enumerate(all_modules):
1214         current_module_fullname = '.'.join(all_modules[:index + 1])
1215         if (current_module_fullname == fullname and not
1216             self.StubModuleExists(fullname)):
1217
1218
1219
1220
1221
1222
1223           self.FindModuleRestricted(current_module,
1224                                     current_module_fullname,
1225                                     search_path)
1226         else:
1227
1228
1229
1230           if current_module_fullname in self._module_dict:
1231             module = self._module_dict[current_module_fullname]
1232           else:
1233
1234             module = self.FindAndLoadModule(current_module,
1235                                             current_module_fullname,
1236                                             search_path)
1237
1238
1239
1240
1241
1242
1243
1244           if hasattr(module, '__path__'):
1245             search_path = module.__path__
1246     except CouldNotFindModuleError:
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256       return None
1257
1258     except Py27OptionalModuleError, err:
1259
1260
1261
1262
1263       logging.error(err)
1264       raise
1265
1266
1267
1268     return self
1269
1270   def StubModuleExists(self, name):
1271     """Check if the named module has a stub replacement."""
1272     if name in sys.builtin_module_names:
1273       name = 'py_%s' % name
1274     if self._config and self._config.runtime == 'python27':
1275       if name in dist27.MODULE_OVERRIDES:
1276         return True
1277     else:
1278       if name in dist.__all__:
1279         return True
1280     return False
1281
1282   def ImportStubModule(self, name):
1283     """Import the stub module replacement for the specified module."""
1284     if name in sys.builtin_module_names:
1285       name = 'py_%s' % name
1286
1287
1288     providing_dist = dist
1289     if self._config and self._config.runtime == 'python27':
1290
1291
1292       if name in dist27.__all__:
1293         providing_dist = dist27
1294     fullname = '%s.%s' % (providing_dist.__name__, name)
1295     __import__(fullname, {}, {})
1296     return sys.modules[fullname]
1297
1298   @Trace
1299   def FixModule(self, module):
1300     """Prunes and overrides restricted module attributes.
1301
1302     Args:
1303       module: The module to prune. This should be a new module whose attributes
1304         reference back to the real module's __dict__ members.
1305     """
1306
1307     if module.__name__ in self._white_list_partial_modules:
1308       allowed_symbols = self._white_list_partial_modules[module.__name__]
1309       for symbol in set(module.__dict__) - set(allowed_symbols):
1310         if not (symbol.startswith('__') and symbol.endswith('__')):
1311           del module.__dict__[symbol]
1312
1313
1314     if module.__name__ in self._MODULE_OVERRIDES:
1315       module.__dict__.update(self._MODULE_OVERRIDES[module.__name__])
1316
1317     if module.__name__ == 'urllib' and NeedsMacOSXProxyFakes():
1318       module.__dict__.update(
1319           {'proxy_bypass_macosx_sysconf': FakeProxyBypassMacOSXSysconf,
1320            'getproxies_macosx_sysconf': FakeGetProxiesMacOSXSysconf})
1321
1322   @Trace
1323   def FindModuleRestricted(self,
1324                            submodule,
1325                            submodule_fullname,
1326                            search_path):
1327     """Locates a module while enforcing module import restrictions.
1328
1329     Args:
1330       submodule: The short name of the submodule (i.e., the last section of
1331         the fullname; for 'foo.bar' this would be 'bar').
1332       submodule_fullname: The fully qualified name of the module to find (e.g.,
1333         'foo.bar').
1334       search_path: List of paths to search for to find this module. Should be
1335         None if the current sys.path should be used.
1336
1337     Returns:
1338       Tuple (source_file, pathname, description) where:
1339         source_file: File-like object that contains the module; in the case
1340           of packages, this will be None, which implies to look at __init__.py.
1341         pathname: String containing the full path of the module on disk.
1342         description: Tuple returned by imp.find_module().
1343       However, in the case of an import using a path hook (e.g. a zipfile),
1344       source_file will be a PEP-302-style loader object, pathname will be None,
1345       and description will be a tuple filled with None values.
1346
1347     Raises:
1348       ImportError exception if the requested module was found, but importing
1349       it is disallowed.
1350
1351       CouldNotFindModuleError exception if the request module could not even
1352       be found for import.
1353     """
1354     if search_path is None:
1355
1356
1357       search_path = [None] + sys.path
1358
1359     py27_optional = False
1360     py27_enabled = False
1361     topmodule = None
1362     if self._config and self._config.runtime == 'python27':
1363
1364
1365       topmodule = submodule_fullname.split('.')[0]
1366       if topmodule in self.__PY27_OPTIONAL_ALLOWED_MODULES:
1367         py27_optional = True
1368         py27_enabled = topmodule in self._enabled_modules
1369
1370
1371
1372       elif topmodule == 'Crypto':
1373         py27_optional = True
1374         py27_enabled = 'pycrypto' in self._enabled_modules
1375
1376
1377
1378
1379
1380
1381     import_error = None
1382     for path_entry in search_path:
1383       result = self.FindPathHook(submodule, submodule_fullname, path_entry)
1384       if result is not None:
1385         source_file, pathname, description = result
1386         if description == (None, None, None):
1387
1388           return result
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409         suffix, mode, file_type = description
1410
1411         try:
1412           if (file_type not in (self._imp.C_BUILTIN, self._imp.C_EXTENSION)):
1413             pkg_pathname = pathname
1414             if file_type == self._imp.PKG_DIRECTORY:
1415
1416
1417               pkg_pathname = os.path.join(pkg_pathname, '__init__.py')
1418             if not FakeFile.IsFileAccessible(
1419                 pkg_pathname, py27_optional=py27_optional):
1420               error_message = 'Access to module file denied: %s' % pathname
1421               logging.debug(error_message)
1422               raise ImportError(error_message)
1423
1424           if (file_type not in self._ENABLED_FILE_TYPES and
1425               submodule not in self._white_list_c_modules):
1426             error_message = ('Could not import "%s": Disallowed C-extension '
1427                              'or built-in module' % submodule_fullname)
1428             logging.debug(error_message)
1429             raise ImportError(error_message)
1430
1431           if (py27_optional and not py27_enabled and
1432               not pathname.startswith(self._app_code_path)):
1433             error_message = ('Third party package %s not enabled.' % topmodule)
1434             logging.debug(error_message)
1435             raise ImportError(error_message)
1436
1437           return source_file, pathname, description
1438         except ImportError, e:
1439
1440
1441           import_error = e
1442
1443
1444
1445     if py27_optional and submodule_fullname == topmodule:
1446       if py27_enabled:
1447         msg = ('Third party package %s was enabled in app.yaml '
1448                'but not found on import. You may have to download '
1449                'and install it.' % topmodule)
1450       else:
1451         msg = ('Third party package %s must be included in the '
1452                '"libraries:" clause of your app.yaml file '
1453                'in order to be imported.' % topmodule)
1454
1455
1456
1457       logging.debug(msg)
1458       raise Py27OptionalModuleError(msg)
1459
1460     if import_error:
1461
1462
1463
1464
1465       raise import_error
1466
1467
1468
1469
1470     self.log('Could not find module "%s"', submodule_fullname)
1471     raise CouldNotFindModuleError()
1472
1473
1474   def FindPathHook(self, submodule, submodule_fullname, path_entry):
1475     """Helper for FindModuleRestricted to find a module in a sys.path entry.
1476
1477     Args:
1478       submodule:
1479       submodule_fullname:
1480       path_entry: A single sys.path entry, or None representing the builtins.
1481
1482     Returns:
1483       Either None (if nothing was found), or a triple (source_file, path_name,
1484       description).  See the doc string for FindModuleRestricted() for the
1485       meaning of the latter.
1486     """
1487     if path_entry is None:
1488
1489       if submodule_fullname in sys.builtin_module_names:
1490         try:
1491           result = self._imp.find_module(submodule)
1492         except ImportError:
1493           pass
1494         else:
1495
1496           source_file, pathname, description = result
1497           suffix, mode, file_type = description
1498           if file_type == self._imp.C_BUILTIN:
1499             return result
1500
1501       return None
1502
1503
1504
1505
1506
1507     if path_entry in sys.path_importer_cache:
1508       importer = sys.path_importer_cache[path_entry]
1509     else:
1510
1511       importer = None
1512       for hook in sys.path_hooks:
1513         try:
1514           importer = hook(path_entry)
1515
1516           break
1517         except ImportError:
1518
1519           pass
1520
1521       sys.path_importer_cache[path_entry] = importer
1522
1523     if importer is None:
1524
1525       try:
1526         return self._imp.find_module(submodule, [path_entry])
1527       except ImportError:
1528         pass
1529     else:
1530
1531       loader = importer.find_module(submodule_fullname)
1532       if loader is not None:
1533
1534
1535
1536
1537         return (loader, None, (None, None, None))
1538
1539
1540     return None
1541
1542   @Trace
1543   def LoadModuleRestricted(self,
1544                            submodule_fullname,
1545                            source_file,
1546                            pathname,
1547                            description):
1548     """Loads a module while enforcing module import restrictions.
1549
1550     As a byproduct, the new module will be added to the module dictionary.
1551
1552     Args:
1553       submodule_fullname: The fully qualified name of the module to find (e.g.,
1554         'foo.bar').
1555       source_file: File-like object that contains the module's source code,
1556         or a PEP-302-style loader object.
1557       pathname: String containing the full path of the module on disk.
1558       description: Tuple returned by imp.find_module(), or (None, None, None)
1559         in case source_file is a PEP-302-style loader object.
1560
1561     Returns:
1562       The new module.
1563
1564     Raises:
1565       ImportError exception of the specified module could not be loaded for
1566       whatever reason.
1567     """
1568     if description == (None, None, None):
1569
1570
1571       return source_file.load_module(submodule_fullname)
1572
1573     try:
1574       try:
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586         return self._imp.load_module(submodule_fullname,
1587                                      source_file,
1588                                      pathname,
1589                                      description)
1590       except:
1591
1592
1593         if submodule_fullname in self._module_dict:
1594           del self._module_dict[submodule_fullname]
1595         raise
1596
1597     finally:
1598       if source_file is not None:
1599         source_file.close()
1600
1601   @Trace
1602   def FindAndLoadModule(self,
1603                         submodule,
1604                         submodule_fullname,
1605                         search_path):
1606     """Finds and loads a module, loads it, and adds it to the module dictionary.
1607
1608     Args:
1609       submodule: Name of the module to import (e.g., baz).
1610       submodule_fullname: Full name of the module to import (e.g., foo.bar.baz).
1611       search_path: Path to use for searching for this submodule. For top-level
1612         modules this should be None; otherwise it should be the __path__
1613         attribute from the parent package.
1614
1615     Returns:
1616       A new module instance that has been inserted into the module dictionary
1617       supplied to __init__.
1618
1619     Raises:
1620       ImportError exception if the module could not be loaded for whatever
1621       reason (e.g., missing, not allowed).
1622     """
1623     module = self._imp.new_module(submodule_fullname)
1624
1625     if submodule_fullname == 'thread':
1626       module.__dict__.update(self._dummy_thread.__dict__)
1627       module.__name__ = 'thread'
1628     elif submodule_fullname == 'cPickle':
1629       module.__dict__.update(self._pickle.__dict__)
1630       module.__name__ = 'cPickle'
1631     elif submodule_fullname == 'os':
1632       module.__dict__.update(self._os.__dict__)
1633     elif submodule_fullname == 'ssl':
1634       pass
1635     elif self.StubModuleExists(submodule_fullname):
1636       module = self.ImportStubModule(submodule_fullname)
1637     else:
1638       source_file, pathname, description = self.FindModuleRestricted(submodule, submodule_fullname, search_path)
1639       module = self.LoadModuleRestricted(submodule_fullname,
1640                                          source_file,
1641                                          pathname,
1642                                          description)
1643
1644
1645
1646
1647     if (getattr(module, '__path__', None) is not None and
1648         search_path != self._app_code_path):
1649       try:
1650         app_search_path = os.path.join(self._app_code_path,
1651                                        *(submodule_fullname.split('.')[:-1]))
1652         source_file, pathname, description = self.FindModuleRestricted(submodule,
1653                                     submodule_fullname,
1654                                     [app_search_path])
1655
1656
1657         module.__path__.append(pathname)
1658       except ImportError, e:
1659         pass
1660
1661
1662
1663
1664     module.__loader__ = self
1665     self.FixModule(module)
1666     if submodule_fullname not in self._module_dict:
1667       self._module_dict[submodule_fullname] = module
1668     if submodule_fullname != submodule:
1669       parent_module = self._module_dict.get(
1670           submodule_fullname[:-len(submodule) - 1])
1671
1672
1673       if parent_module and not hasattr(parent_module, submodule):
1674         setattr(parent_module, submodule, module)
1675
1676     if submodule_fullname == 'os':
1677
1678
1679
1680
1681
1682
1683
1684
1685       os_path_name = module.path.__name__
1686       os_path = self.FindAndLoadModule(os_path_name, os_path_name, search_path)
1687
1688
1689       self._module_dict['os.path'] = os_path
1690       module.__dict__['path'] = os_path
1691
1692     return module
1693
1694   @Trace
1695   def GetParentPackage(self, fullname):
1696     """Retrieves the parent package of a fully qualified module name.
1697
1698     Args:
1699       fullname: Full name of the module whose parent should be retrieved (e.g.,
1700         foo.bar).
1701
1702     Returns:
1703       Module instance for the parent or None if there is no parent module.
1704
1705     Raise:
1706       ImportError exception if the module's parent could not be found.
1707     """
1708     all_modules = fullname.split('.')
1709     parent_module_fullname = '.'.join(all_modules[:-1])
1710     if parent_module_fullname:
1711
1712       if self.find_module(fullname) is None:
1713         raise ImportError('Could not find module %s' % fullname)
1714
1715       return self._module_dict[parent_module_fullname]
1716     return None
1717
1718   @Trace
1719   def GetParentSearchPath(self, fullname):
1720     """Determines the search path of a module's parent package.
1721
1722     Args:
1723       fullname: Full name of the module to look up (e.g., foo.bar).
1724
1725     Returns:
1726       Tuple (submodule, search_path) where:
1727         submodule: The last portion of the module name from fullname (e.g.,
1728           if fullname is foo.bar, then this is bar).
1729         search_path: List of paths that belong to the parent package's search
1730           path or None if there is no parent package.
1731
1732     Raises:
1733       ImportError exception if the module or its parent could not be found.
1734     """
1735     submodule = GetSubmoduleName(fullname)
1736     parent_package = self.GetParentPackage(fullname)
1737     search_path = None
1738     if parent_package is not None and hasattr(parent_package, '__path__'):
1739       search_path = parent_package.__path__
1740     return submodule, search_path
1741
1742   @Trace
1743   def GetModuleInfo(self, fullname):
1744     """Determines the path on disk and the search path of a module or package.
1745
1746     Args:
1747       fullname: Full name of the module to look up (e.g., foo.bar).
1748
1749     Returns:
1750       Tuple (pathname, search_path, submodule) where:
1751         pathname: String containing the full path of the module on disk,
1752           or None if the module wasn't loaded from disk (e.g. from a zipfile).
1753         search_path: List of paths that belong to the found package's search
1754           path or None if found module is not a package.
1755         submodule: The relative name of the submodule that's being imported.
1756     """
1757     submodule, search_path = self.GetParentSearchPath(fullname)
1758     source_file, pathname, description = self.FindModuleRestricted(submodule, fullname, search_path)
1759     suffix, mode, file_type = description
1760     module_search_path = None
1761     if file_type == self._imp.PKG_DIRECTORY:
1762       module_search_path = [pathname]
1763       pathname = os.path.join(pathname, '__init__%spy' % os.extsep)
1764     return pathname, module_search_path, submodule
1765
1766   @Trace
1767   def load_module(self, fullname):
1768     """See PEP 302."""
1769     all_modules = fullname.split('.')
1770     submodule = all_modules[-1]
1771     parent_module_fullname = '.'.join(all_modules[:-1])
1772     search_path = None
1773     if parent_module_fullname and parent_module_fullname in self._module_dict:
1774       parent_module = self._module_dict[parent_module_fullname]
1775       if hasattr(parent_module, '__path__'):
1776         search_path = parent_module.__path__
1777
1778     return self.FindAndLoadModule(submodule, fullname, search_path)
1779
1780   @Trace
1781   def is_package(self, fullname):
1782     """See PEP 302 extensions."""
1783     submodule, search_path = self.GetParentSearchPath(fullname)
1784     source_file, pathname, description = self.FindModuleRestricted(submodule, fullname, search_path)
1785     suffix, mode, file_type = description
1786     if file_type == self._imp.PKG_DIRECTORY:
1787       return True
1788     return False
1789
1790   @Trace
1791   def get_source(self, fullname):
1792     """See PEP 302 extensions."""
1793     full_path, search_path, submodule = self.GetModuleInfo(fullname)
1794     if full_path is None:
1795       return None
1796     source_file = open(full_path)
1797     try:
1798       return source_file.read()
1799     finally:
1800       source_file.close()
1801
1802   @Trace
1803   def get_code(self, fullname):
1804     """See PEP 302 extensions."""
1805     full_path, search_path, submodule = self.GetModuleInfo(fullname)
1806     if full_path is None:
1807       return None
1808     source_file = open(full_path)
1809     try:
1810       source_code = source_file.read()
1811     finally:
1812       source_file.close()
1813
1814
1815
1816
1817     source_code = source_code.replace('\r\n', '\n')
1818     if not source_code.endswith('\n'):
1819       source_code += '\n'
1820
1821
1822
1823
1824     encoding = DEFAULT_ENCODING
1825     for line in source_code.split('\n', 2)[:2]:
1826       matches = CODING_COOKIE_RE.findall(line)
1827       if matches:
1828         encoding = matches[0].lower()
1829
1830
1831     source_code.decode(encoding)
1832
1833     return compile(source_code, full_path, 'exec')