3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """Pure-Python application server for testing applications locally.
19 Given a port and the paths to a valid application directory (with an 'app.yaml'
20 file), the external library directory, and a relative URL to use for logins,
21 creates an HTTP server that can be used to test an application locally. Uses
22 stubs instead of actual APIs when SetupStubs() is called first.
25 root_path = '/path/to/application/directory'
28 template_dir = '/path/to/appserver/templates'
29 server = dev_appserver.CreateServer(root_path, login_url, port, template_dir)
30 server.serve_forever()
34 from google
.appengine
.tools
import os_compat
79 from google
.pyglib
import gexcept
81 from google
.appengine
.api
import apiproxy_stub_map
82 from google
.appengine
.api
import appinfo
83 from google
.appengine
.api
import croninfo
84 from google
.appengine
.api
import datastore_admin
85 from google
.appengine
.api
import datastore_file_stub
86 from google
.appengine
.api
import mail_stub
87 from google
.appengine
.api
import urlfetch_stub
88 from google
.appengine
.api
import user_service_stub
89 from google
.appengine
.api
import yaml_errors
90 from google
.appengine
.api
.capabilities
import capability_stub
91 from google
.appengine
.api
.memcache
import memcache_stub
93 from google
.appengine
import dist
95 from google
.appengine
.tools
import dev_appserver_index
96 from google
.appengine
.tools
import dev_appserver_login
99 PYTHON_LIB_VAR
= '$PYTHON_LIB'
100 DEVEL_CONSOLE_PATH
= PYTHON_LIB_VAR
+ '/google/appengine/ext/admin'
102 FILE_MISSING_EXCEPTIONS
= frozenset([errno
.ENOENT
, errno
.ENOTDIR
])
104 MAX_URL_LENGTH
= 2047
106 HEADER_TEMPLATE
= 'logging_console_header.html'
107 SCRIPT_TEMPLATE
= 'logging_console.js'
108 MIDDLE_TEMPLATE
= 'logging_console_middle.html'
109 FOOTER_TEMPLATE
= 'logging_console_footer.html'
112 'GATEWAY_INTERFACE': 'CGI/1.1',
113 'AUTH_DOMAIN': 'gmail.com',
117 for ext
, mime_type
in (('.asc', 'text/plain'),
118 ('.diff', 'text/plain'),
119 ('.csv', 'text/comma-separated-values'),
120 ('.rss', 'application/rss+xml'),
121 ('.text', 'text/plain'),
122 ('.wbmp', 'image/vnd.wap.wbmp')):
123 mimetypes
.add_type(mime_type
, ext
)
125 MAX_RUNTIME_RESPONSE_SIZE
= 10 << 20
127 MAX_REQUEST_SIZE
= 10 * 1024 * 1024
132 class Error(Exception):
133 """Base-class for exceptions in this module."""
135 class InvalidAppConfigError(Error
):
136 """The supplied application configuration file is invalid."""
138 class AppConfigNotFoundError(Error
):
139 """Application configuration file not found."""
141 class TemplatesNotLoadedError(Error
):
142 """Templates for the debugging console were not loaded."""
145 def SplitURL(relative_url
):
146 """Splits a relative URL into its path and query-string components.
149 relative_url: String containing the relative URL (often starting with '/')
150 to split. Should be properly escaped as www-form-urlencoded data.
153 Tuple (script_name, query_string) where:
154 script_name: Relative URL of the script that was accessed.
155 query_string: String containing everything after the '?' character.
157 scheme
, netloc
, path
, query
, fragment
= urlparse
.urlsplit(relative_url
)
161 def GetFullURL(server_name
, server_port
, relative_url
):
162 """Returns the full, original URL used to access the relative URL.
165 server_name: Name of the local host, or the value of the 'host' header
167 server_port: Port on which the request was served (string or int).
168 relative_url: Relative URL that was accessed, including query string.
171 String containing the original URL.
173 if str(server_port
) != '80':
174 netloc
= '%s:%s' % (server_name
, server_port
)
177 return 'http://%s%s' % (netloc
, relative_url
)
180 class URLDispatcher(object):
181 """Base-class for handling HTTP requests."""
190 """Dispatch and handle an HTTP request.
192 base_env_dict should contain at least these CGI variables:
193 REQUEST_METHOD, REMOTE_ADDR, SERVER_SOFTWARE, SERVER_NAME,
194 SERVER_PROTOCOL, SERVER_PORT
197 relative_url: String containing the URL accessed.
198 path: Local path of the resource that was matched; back-references will be
199 replaced by values matched in the relative_url. Path may be relative
200 or absolute, depending on the resource being served (e.g., static files
201 will have an absolute path; scripts will be relative).
202 headers: Instance of mimetools.Message with headers from the request.
203 infile: File-like object with input data from the request.
204 outfile: File-like object where output data should be written.
205 base_env_dict: Dictionary of CGI environment parameters if available.
209 None if request handling is complete.
210 Tuple (path, headers, input_file) for an internal redirect:
211 path: Path of URL to redirect to.
212 headers: Headers to send to other dispatcher.
213 input_file: New input to send to new dispatcher.
215 raise NotImplementedError
217 def EndRedirect(self
, dispatched_output
, original_output
):
218 """Process the end of an internal redirect.
220 This method is called after all subsequent dispatch requests have finished.
221 By default the output from the dispatched process is copied to the original.
223 This will not be called on dispatchers that do not return an internal
227 dispatched_output: StringIO buffer containing the results from the
230 original_output
.write(dispatched_output
.read())
233 class URLMatcher(object):
234 """Matches an arbitrary URL using a list of URL patterns from an application.
236 Each URL pattern has an associated URLDispatcher instance and path to the
237 resource's location on disk. See AddURL for more details. The first pattern
238 that matches an inputted URL will have its associated values returned by
244 self
._url
_patterns
= []
246 def AddURL(self
, regex
, dispatcher
, path
, requires_login
, admin_only
):
247 """Adds a URL pattern to the list of patterns.
249 If the supplied regex starts with a '^' or ends with a '$' an
250 InvalidAppConfigError exception will be raised. Start and end symbols
251 and implicitly added to all regexes, meaning we assume that all regexes
252 consume all input from a URL.
255 regex: String containing the regular expression pattern.
256 dispatcher: Instance of URLDispatcher that should handle requests that
258 path: Path on disk for the resource. May contain back-references like
259 r'\1', r'\2', etc, which will be replaced by the corresponding groups
260 matched by the regex if present.
261 requires_login: True if the user must be logged-in before accessing this
262 URL; False if anyone can access this URL.
263 admin_only: True if the user must be a logged-in administrator to
264 access the URL; False if anyone can access the URL.
266 if not isinstance(dispatcher
, URLDispatcher
):
267 raise TypeError('dispatcher must be a URLDispatcher sub-class')
269 if regex
.startswith('^') or regex
.endswith('$'):
270 raise InvalidAppConfigError('regex starts with "^" or ends with "$"')
272 adjusted_regex
= '^%s$' % regex
275 url_re
= re
.compile(adjusted_regex
)
277 raise InvalidAppConfigError('regex invalid: %s' % e
)
279 match_tuple
= (url_re
, dispatcher
, path
, requires_login
, admin_only
)
280 self
._url
_patterns
.append(match_tuple
)
285 """Matches a URL from a request against the list of URL patterns.
287 The supplied relative_url may include the query string (i.e., the '?'
288 character and everything following).
291 relative_url: Relative URL being accessed in a request.
294 Tuple (dispatcher, matched_path, requires_login, admin_only), which are
295 the corresponding values passed to AddURL when the matching URL pattern
296 was added to this matcher. The matched_path will have back-references
297 replaced using values matched by the URL pattern. If no match was found,
298 dispatcher will be None.
300 adjusted_url
, query_string
= split_url(relative_url
)
302 for url_tuple
in self
._url
_patterns
:
303 url_re
, dispatcher
, path
, requires_login
, admin_only
= url_tuple
304 the_match
= url_re
.match(adjusted_url
)
307 adjusted_path
= the_match
.expand(path
)
308 return dispatcher
, adjusted_path
, requires_login
, admin_only
310 return None, None, None, None
312 def GetDispatchers(self
):
313 """Retrieves the URLDispatcher objects that could be matched.
315 Should only be used in tests.
318 A set of URLDispatcher objects.
320 return set([url_tuple
[1] for url_tuple
in self
._url
_patterns
])
323 class MatcherDispatcher(URLDispatcher
):
324 """Dispatcher across multiple URLMatcher instances."""
329 get_user_info
=dev_appserver_login
.GetUserInfo
,
330 login_redirect
=dev_appserver_login
.LoginRedirect
):
334 login_url: Relative URL which should be used for handling user logins.
335 url_matchers: Sequence of URLMatcher objects.
336 get_user_info, login_redirect: Used for dependency injection.
338 self
._login
_url
= login_url
339 self
._url
_matchers
= tuple(url_matchers
)
340 self
._get
_user
_info
= get_user_info
341 self
._login
_redirect
= login_redirect
350 """Dispatches a request to the first matching dispatcher.
352 Matchers are checked in the order they were supplied to the constructor.
353 If no matcher matches, a 404 error will be written to the outfile. The
354 path variable supplied to this method is ignored.
356 cookies
= ', '.join(headers
.getheaders('cookie'))
357 email
, admin
, user_id
= self
._get
_user
_info
(cookies
)
359 for matcher
in self
._url
_matchers
:
360 dispatcher
, matched_path
, requires_login
, admin_only
= matcher
.Match(relative_url
)
361 if dispatcher
is None:
364 logging
.debug('Matched "%s" to %s with path %s',
365 relative_url
, dispatcher
, matched_path
)
367 if (requires_login
or admin_only
) and not email
:
368 logging
.debug('Login required, redirecting user')
369 self
._login
_redirect
(
371 base_env_dict
['SERVER_NAME'],
372 base_env_dict
['SERVER_PORT'],
375 elif admin_only
and not admin
:
376 outfile
.write('Status: %d Not authorized\r\n'
378 'Current logged in user %s is not '
379 'authorized to view this page.'
380 % (httplib
.FORBIDDEN
, email
))
382 forward
= dispatcher
.Dispatch(relative_url
,
387 base_env_dict
=base_env_dict
)
390 new_path
, new_headers
, new_input
= forward
391 logging
.info('Internal redirection to %s' % new_path
)
392 new_outfile
= cStringIO
.StringIO()
393 self
.Dispatch(new_path
,
400 dispatcher
.EndRedirect(new_outfile
, outfile
)
404 outfile
.write('Status: %d URL did not match\r\n'
406 'Not found error: %s did not match any patterns '
407 'in application configuration.'
408 % (httplib
.NOT_FOUND
, relative_url
))
411 class ApplicationLoggingHandler(logging
.Handler
):
412 """Python Logging handler that displays the debugging console to users."""
414 _COOKIE_NAME
= '_ah_severity'
416 _TEMPLATES_INITIALIZED
= False
423 def InitializeTemplates(header
, script
, middle
, footer
):
424 """Initializes the templates used to render the debugging console.
426 This method must be called before any ApplicationLoggingHandler instances
430 header: The header template that is printed first.
431 script: The script template that is printed after the logging messages.
432 middle: The middle element that's printed before the footer.
433 footer; The last element that's printed at the end of the document.
435 ApplicationLoggingHandler
._HEADER
= header
436 ApplicationLoggingHandler
._SCRIPT
= script
437 ApplicationLoggingHandler
._MIDDLE
= middle
438 ApplicationLoggingHandler
._FOOTER
= footer
439 ApplicationLoggingHandler
._TEMPLATES
_INITIALIZED
= True
442 def AreTemplatesInitialized():
443 """Returns True if InitializeTemplates has been called, False otherwise."""
444 return ApplicationLoggingHandler
._TEMPLATES
_INITIALIZED
446 def __init__(self
, *args
, **kwargs
):
450 args, kwargs: See logging.Handler.
453 TemplatesNotLoadedError exception if the InitializeTemplates method was
454 not called before creating this instance.
456 if not self
._TEMPLATES
_INITIALIZED
:
457 raise TemplatesNotLoadedError
459 logging
.Handler
.__init
__(self
, *args
, **kwargs
)
460 self
._record
_list
= []
461 self
._start
_time
= time
.time()
463 def emit(self
, record
):
464 """Called by the logging module each time the application logs a message.
467 record: logging.LogRecord instance corresponding to the newly logged
470 self
._record
_list
.append(record
)
472 def AddDebuggingConsole(self
, relative_url
, env
, outfile
):
473 """Prints an HTML debugging console to an output stream, if requested.
476 relative_url: Relative URL that was accessed, including the query string.
477 Used to determine if the parameter 'debug' was supplied, in which case
478 the console will be shown.
479 env: Dictionary containing CGI environment variables. Checks for the
480 HTTP_COOKIE entry to see if the accessing user has any logging-related
482 outfile: Output stream to which the console should be written if either
483 a debug parameter was supplied or a logging cookie is present.
485 script_name
, query_string
= SplitURL(relative_url
)
486 param_dict
= cgi
.parse_qs(query_string
, True)
487 cookie_dict
= Cookie
.SimpleCookie(env
.get('HTTP_COOKIE', ''))
488 if 'debug' not in param_dict
and self
._COOKIE
_NAME
not in cookie_dict
:
491 outfile
.write(self
._HEADER
)
492 for record
in self
._record
_list
:
493 self
._PrintRecord
(record
, outfile
)
495 outfile
.write(self
._MIDDLE
)
496 outfile
.write(self
._SCRIPT
)
497 outfile
.write(self
._FOOTER
)
499 def _PrintRecord(self
, record
, outfile
):
500 """Prints a single logging record to an output stream.
503 record: logging.LogRecord instance to print.
504 outfile: Output stream to which the LogRecord should be printed.
506 message
= cgi
.escape(record
.getMessage())
507 level_name
= logging
.getLevelName(record
.levelno
).lower()
508 level_letter
= level_name
[:1].upper()
509 time_diff
= record
.created
- self
._start
_time
510 outfile
.write('<span class="_ah_logline_%s">\n' % level_name
)
511 outfile
.write('<span class="_ah_logline_%s_prefix">%2.5f %s ></span>\n'
512 % (level_name
, time_diff
, level_letter
))
513 outfile
.write('%s\n' % message
)
514 outfile
.write('</span>\n')
517 _IGNORE_REQUEST_HEADERS
= frozenset(['content-type', 'content-length',
518 'accept-encoding', 'transfer-encoding'])
520 def SetupEnvironment(cgi_path
,
524 get_user_info
=dev_appserver_login
.GetUserInfo
):
525 """Sets up environment variables for a CGI.
528 cgi_path: Full file-system path to the CGI being executed.
529 relative_url: Relative URL used to access the CGI.
530 headers: Instance of mimetools.Message containing request headers.
531 split_url, get_user_info: Used for dependency injection.
534 Dictionary containing CGI environment variables.
536 env
= DEFAULT_ENV
.copy()
538 script_name
, query_string
= split_url(relative_url
)
540 env
['SCRIPT_NAME'] = ''
541 env
['QUERY_STRING'] = query_string
542 env
['PATH_INFO'] = urllib
.unquote(script_name
)
543 env
['PATH_TRANSLATED'] = cgi_path
544 env
['CONTENT_TYPE'] = headers
.getheader('content-type',
545 'application/x-www-form-urlencoded')
546 env
['CONTENT_LENGTH'] = headers
.getheader('content-length', '')
548 cookies
= ', '.join(headers
.getheaders('cookie'))
549 email
, admin
, user_id
= get_user_info(cookies
)
550 env
['USER_EMAIL'] = email
551 env
['USER_ID'] = user_id
553 env
['USER_IS_ADMIN'] = '1'
556 if key
in _IGNORE_REQUEST_HEADERS
:
558 adjusted_name
= key
.replace('-', '_').upper()
559 env
['HTTP_' + adjusted_name
] = ', '.join(headers
.getheaders(key
))
564 def NotImplementedFake(*args
, **kwargs
):
565 """Fake for methods/functions that are not implemented in the production
568 raise NotImplementedError("This class/method is not available.")
571 class NotImplementedFakeClass(object):
572 """Fake class for classes that are not implemented in the production
575 __init__
= NotImplementedFake
578 def IsEncodingsModule(module_name
):
579 """Determines if the supplied module is related to encodings in any way.
581 Encodings-related modules cannot be reloaded, so they need to be treated
582 specially when sys.modules is modified in any way.
585 module_name: Absolute name of the module regardless of how it is imported
586 into the local namespace (e.g., foo.bar.baz).
589 True if it's an encodings-related module; False otherwise.
591 if (module_name
in ('codecs', 'encodings') or
592 module_name
.startswith('encodings.')):
597 def ClearAllButEncodingsModules(module_dict
):
598 """Clear all modules in a module dictionary except for those modules that
599 are in any way related to encodings.
602 module_dict: Dictionary in the form used by sys.modules.
604 for module_name
in module_dict
.keys():
605 if not IsEncodingsModule(module_name
):
606 del module_dict
[module_name
]
610 """Fake version of os.urandom."""
613 bytes
+= chr(random
.randint(0, 255))
618 """Fake version of os.uname."""
619 return ('Linux', '', '', '', '')
622 def FakeUnlink(path
):
623 """Fake version of os.unlink."""
624 if os
.path
.isdir(path
):
625 raise OSError(errno
.ENOENT
, "Is a directory", path
)
627 raise OSError(errno
.EPERM
, "Operation not permitted", path
)
630 def FakeReadlink(path
):
631 """Fake version of os.readlink."""
632 raise OSError(errno
.EINVAL
, "Invalid argument", path
)
635 def FakeAccess(path
, mode
):
636 """Fake version of os.access where only reads are supported."""
637 if not os
.path
.exists(path
) or mode
!= os
.R_OK
:
643 def FakeSetLocale(category
, value
=None, original_setlocale
=locale
.setlocale
):
644 """Fake version of locale.setlocale that only supports the default."""
645 if value
not in (None, '', 'C', 'POSIX'):
646 raise locale
.Error('locale emulation only supports "C" locale')
647 return original_setlocale(category
, 'C')
650 def FakeOpen(file, flags
, mode
=0777):
651 """Fake version of os.open."""
652 raise OSError(errno
.EPERM
, "Operation not permitted", file)
655 def FakeRename(src
, dst
):
656 """Fake version of os.rename."""
657 raise OSError(errno
.EPERM
, "Operation not permitted", src
)
660 def FakeUTime(path
, times
):
661 """Fake version of os.utime."""
662 raise OSError(errno
.EPERM
, "Operation not permitted", path
)
665 def FakeGetPlatform():
666 """Fake distutils.util.get_platform on OS/X. Pass-through otherwise."""
667 if sys
.platform
== 'darwin':
670 return distutils
.util
.get_platform()
673 def IsPathInSubdirectories(filename
,
675 normcase
=os
.path
.normcase
):
676 """Determines if a filename is contained within one of a set of directories.
679 filename: Path of the file (relative or absolute).
680 subdirectories: Iterable collection of paths to subdirectories which the
681 given filename may be under.
682 normcase: Used for dependency injection.
685 True if the supplied filename is in one of the given sub-directories or
686 its hierarchy of children. False otherwise.
688 file_dir
= normcase(os
.path
.dirname(os
.path
.abspath(filename
)))
689 for parent
in subdirectories
:
690 fixed_parent
= normcase(os
.path
.abspath(parent
))
691 if os
.path
.commonprefix([file_dir
, fixed_parent
]) == fixed_parent
:
695 SHARED_MODULE_PREFIXES
= set([
715 NOT_SHARED_MODULE_PREFIXES
= set([
716 'google.appengine.ext',
720 def ModuleNameHasPrefix(module_name
, prefix_set
):
721 """Determines if a module's name belongs to a set of prefix strings.
724 module_name: String containing the fully qualified module name.
725 prefix_set: Iterable set of module name prefixes to check against.
728 True if the module_name belongs to the prefix set or is a submodule of
729 any of the modules specified in the prefix_set. Otherwise False.
731 for prefix
in prefix_set
:
732 if prefix
== module_name
:
735 if module_name
.startswith(prefix
+ '.'):
741 def SetupSharedModules(module_dict
):
742 """Creates a module dictionary for the hardened part of the process.
744 Module dictionary will contain modules that should be shared between the
745 hardened and unhardened parts of the process.
748 module_dict: Module dictionary from which existing modules should be
749 pulled (usually sys.modules).
752 A new module dictionary.
755 for module_name
, module
in module_dict
.iteritems():
759 if IsEncodingsModule(module_name
):
760 output_dict
[module_name
] = module
763 shared_prefix
= ModuleNameHasPrefix(module_name
, SHARED_MODULE_PREFIXES
)
764 banned_prefix
= ModuleNameHasPrefix(module_name
, NOT_SHARED_MODULE_PREFIXES
)
766 if shared_prefix
and not banned_prefix
:
767 output_dict
[module_name
] = module
772 def GeneratePythonPaths(*p
):
773 """Generate all valid filenames for the given file
776 p: Positional args are the folders to the file and finally the file
780 A list of strings representing the given path to a file with each valid
781 suffix for this python build.
783 suffixes
= imp
.get_suffixes()
784 return [os
.path
.join(*p
) + s
for s
, m
, t
in suffixes
]
787 class FakeFile(file):
788 """File sub-class that enforces the security restrictions of the production
792 ALLOWED_MODES
= frozenset(['r', 'rb', 'U', 'rU'])
794 ALLOWED_FILES
= set(os
.path
.normcase(filename
)
795 for filename
in mimetypes
.knownfiles
796 if os
.path
.isfile(filename
))
799 os
.path
.normcase(os
.path
.realpath(os
.path
.dirname(os
.__file
__))),
800 os
.path
.normcase(os
.path
.abspath(os
.path
.dirname(os
.__file
__))),
803 NOT_ALLOWED_DIRS
= set([
808 os
.path
.normcase(os
.path
.join(os
.path
.dirname(os
.__file
__),
812 ALLOWED_SITE_PACKAGE_DIRS
= set(
813 os
.path
.normcase(os
.path
.abspath(os
.path
.join(
814 os
.path
.dirname(os
.__file
__), 'site-packages', path
)))
819 ALLOWED_SITE_PACKAGE_FILES
= set(
820 os
.path
.normcase(os
.path
.abspath(os
.path
.join(
821 os
.path
.dirname(os
.__file
__), 'site-packages', path
)))
822 for path
in itertools
.chain(*[
824 [os
.path
.join('Crypto')],
825 GeneratePythonPaths('Crypto', '__init__'),
826 [os
.path
.join('Crypto', 'Cipher')],
827 GeneratePythonPaths('Crypto', 'Cipher', '__init__'),
828 GeneratePythonPaths('Crypto', 'Cipher', 'AES'),
829 GeneratePythonPaths('Crypto', 'Cipher', 'ARC2'),
830 GeneratePythonPaths('Crypto', 'Cipher', 'ARC4'),
831 GeneratePythonPaths('Crypto', 'Cipher', 'Blowfish'),
832 GeneratePythonPaths('Crypto', 'Cipher', 'CAST'),
833 GeneratePythonPaths('Crypto', 'Cipher', 'DES'),
834 GeneratePythonPaths('Crypto', 'Cipher', 'DES3'),
835 GeneratePythonPaths('Crypto', 'Cipher', 'XOR'),
836 [os
.path
.join('Crypto', 'Hash')],
837 GeneratePythonPaths('Crypto', 'Hash', '__init__'),
838 GeneratePythonPaths('Crypto', 'Hash', 'HMAC'),
839 os
.path
.join('Crypto', 'Hash', 'MD2'),
840 os
.path
.join('Crypto', 'Hash', 'MD4'),
841 GeneratePythonPaths('Crypto', 'Hash', 'MD5'),
842 GeneratePythonPaths('Crypto', 'Hash', 'SHA'),
843 os
.path
.join('Crypto', 'Hash', 'SHA256'),
844 os
.path
.join('Crypto', 'Hash', 'RIPEMD'),
845 [os
.path
.join('Crypto', 'Protocol')],
846 GeneratePythonPaths('Crypto', 'Protocol', '__init__'),
847 GeneratePythonPaths('Crypto', 'Protocol', 'AllOrNothing'),
848 GeneratePythonPaths('Crypto', 'Protocol', 'Chaffing'),
849 [os
.path
.join('Crypto', 'PublicKey')],
850 GeneratePythonPaths('Crypto', 'PublicKey', '__init__'),
851 GeneratePythonPaths('Crypto', 'PublicKey', 'DSA'),
852 GeneratePythonPaths('Crypto', 'PublicKey', 'ElGamal'),
853 GeneratePythonPaths('Crypto', 'PublicKey', 'RSA'),
854 GeneratePythonPaths('Crypto', 'PublicKey', 'pubkey'),
855 GeneratePythonPaths('Crypto', 'PublicKey', 'qNEW'),
856 [os
.path
.join('Crypto', 'Util')],
857 GeneratePythonPaths('Crypto', 'Util', '__init__'),
858 GeneratePythonPaths('Crypto', 'Util', 'RFC1751'),
859 GeneratePythonPaths('Crypto', 'Util', 'number'),
860 GeneratePythonPaths('Crypto', 'Util', 'randpool'),
863 _original_file
= file
866 _application_paths
= None
868 _static_file_config_matcher
= None
870 _allow_skipped_files
= True
872 _availability_cache
= {}
875 def SetAllowedPaths(root_path
, application_paths
):
876 """Configures which paths are allowed to be accessed.
878 Must be called at least once before any file objects are created in the
879 hardened environment.
882 root_path: Absolute path to the root of the application.
883 application_paths: List of additional paths that the application may
884 access, this must include the App Engine runtime but
885 not the Python library directories.
887 FakeFile
._application
_paths
= (set(os
.path
.realpath(path
)
888 for path
in application_paths
) |
889 set(os
.path
.abspath(path
)
890 for path
in application_paths
))
891 FakeFile
._application
_paths
.add(root_path
)
893 FakeFile
._root
_path
= os
.path
.join(root_path
, '')
895 FakeFile
._availability
_cache
= {}
898 def SetAllowSkippedFiles(allow_skipped_files
):
899 """Configures access to files matching FakeFile._skip_files
902 allow_skipped_files: Boolean whether to allow access to skipped files
904 FakeFile
._allow
_skipped
_files
= allow_skipped_files
905 FakeFile
._availability
_cache
= {}
908 def SetSkippedFiles(skip_files
):
909 """Sets which files in the application directory are to be ignored.
911 Must be called at least once before any file objects are created in the
912 hardened environment.
914 Must be called whenever the configuration was updated.
917 skip_files: Object with .match() method (e.g. compiled regexp).
919 FakeFile
._skip
_files
= skip_files
920 FakeFile
._availability
_cache
= {}
923 def SetStaticFileConfigMatcher(static_file_config_matcher
):
924 """Sets StaticFileConfigMatcher instance for checking if a file is static.
926 Must be called at least once before any file objects are created in the
927 hardened environment.
929 Must be called whenever the configuration was updated.
932 static_file_config_matcher: StaticFileConfigMatcher instance.
934 FakeFile
._static
_file
_config
_matcher
= static_file_config_matcher
935 FakeFile
._availability
_cache
= {}
938 def IsFileAccessible(filename
, normcase
=os
.path
.normcase
):
939 """Determines if a file's path is accessible.
941 SetAllowedPaths(), SetSkippedFiles() and SetStaticFileConfigMatcher() must
942 be called before this method or else all file accesses will raise an error.
945 filename: Path of the file to check (relative or absolute). May be a
946 directory, in which case access for files inside that directory will
948 normcase: Used for dependency injection.
951 True if the file is accessible, False otherwise.
953 logical_filename
= normcase(os
.path
.abspath(filename
))
955 result
= FakeFile
._availability
_cache
.get(logical_filename
)
957 result
= FakeFile
._IsFileAccessibleNoCache
(logical_filename
,
959 FakeFile
._availability
_cache
[logical_filename
] = result
963 def _IsFileAccessibleNoCache(logical_filename
, normcase
=os
.path
.normcase
):
964 """Determines if a file's path is accessible.
966 This is an internal part of the IsFileAccessible implementation.
969 logical_filename: Absolute path of the file to check.
970 normcase: Used for dependency injection.
973 True if the file is accessible, False otherwise.
975 logical_dirfakefile
= logical_filename
976 if os
.path
.isdir(logical_filename
):
977 logical_dirfakefile
= os
.path
.join(logical_filename
, 'foo')
979 if IsPathInSubdirectories(logical_dirfakefile
, [FakeFile
._root
_path
],
981 relative_filename
= logical_dirfakefile
[len(FakeFile
._root
_path
):]
983 if (not FakeFile
._allow
_skipped
_files
and
984 FakeFile
._skip
_files
.match(relative_filename
)):
985 logging
.warning('Blocking access to skipped file "%s"',
989 if FakeFile
._static
_file
_config
_matcher
.IsStaticFile(relative_filename
):
990 logging
.warning('Blocking access to static file "%s"',
994 if logical_filename
in FakeFile
.ALLOWED_FILES
:
997 if logical_filename
in FakeFile
.ALLOWED_SITE_PACKAGE_FILES
:
1000 if IsPathInSubdirectories(logical_dirfakefile
,
1001 FakeFile
.ALLOWED_SITE_PACKAGE_DIRS
,
1005 allowed_dirs
= FakeFile
._application
_paths | FakeFile
.ALLOWED_DIRS
1006 if (IsPathInSubdirectories(logical_dirfakefile
,
1008 normcase
=normcase
) and
1009 not IsPathInSubdirectories(logical_dirfakefile
,
1010 FakeFile
.NOT_ALLOWED_DIRS
,
1011 normcase
=normcase
)):
1016 def __init__(self
, filename
, mode
='r', bufsize
=-1, **kwargs
):
1017 """Initializer. See file built-in documentation."""
1018 if mode
not in FakeFile
.ALLOWED_MODES
:
1019 raise IOError('invalid mode: %s' % mode
)
1021 if not FakeFile
.IsFileAccessible(filename
):
1022 raise IOError(errno
.EACCES
, 'file not accessible', filename
)
1024 super(FakeFile
, self
).__init
__(filename
, mode
, bufsize
, **kwargs
)
1027 class RestrictedPathFunction(object):
1028 """Enforces access restrictions for functions that have a file or
1029 directory path as their first argument."""
1033 def __init__(self
, original_func
):
1037 original_func: Callable that takes as its first argument the path to a
1038 file or directory on disk; all subsequent arguments may be variable.
1040 self
._original
_func
= original_func
1042 def __call__(self
, path
, *args
, **kwargs
):
1043 """Enforces access permissions for the function passed to the constructor.
1045 if not FakeFile
.IsFileAccessible(path
):
1046 raise OSError(errno
.EACCES
, 'path not accessible', path
)
1048 return self
._original
_func
(path
, *args
, **kwargs
)
1051 def GetSubmoduleName(fullname
):
1052 """Determines the leaf submodule name of a full module name.
1055 fullname: Fully qualified module name, e.g. 'foo.bar.baz'
1058 Submodule name, e.g. 'baz'. If the supplied module has no submodule (e.g.,
1059 'stuff'), the returned value will just be that module name ('stuff').
1061 return fullname
.rsplit('.', 1)[-1]
1064 class CouldNotFindModuleError(ImportError):
1065 """Raised when a module could not be found.
1067 In contrast to when a module has been found, but cannot be loaded because of
1068 hardening restrictions.
1073 """Decorator that logs the call stack of the HardenedModulesHook class as
1074 it executes, indenting logging messages based on the current stack depth.
1076 def decorate(self
, *args
, **kwargs
):
1078 if args
is not None:
1079 args_to_show
.extend(str(argument
) for argument
in args
)
1080 if kwargs
is not None:
1081 args_to_show
.extend('%s=%s' % (key
, value
)
1082 for key
, value
in kwargs
.iteritems())
1084 args_string
= ', '.join(args_to_show
)
1086 self
.log('Entering %s(%s)', func
.func_name
, args_string
)
1087 self
._indent
_level
+= 1
1089 return func(self
, *args
, **kwargs
)
1091 self
._indent
_level
-= 1
1092 self
.log('Exiting %s(%s)', func
.func_name
, args_string
)
1097 class HardenedModulesHook(object):
1098 """Meta import hook that restricts the modules used by applications to match
1099 the production environment.
1101 Module controls supported:
1102 - Disallow native/extension modules from being loaded
1103 - Disallow built-in and/or Python-distributed modules from being loaded
1104 - Replace modules with completely empty modules
1105 - Override specific module attributes
1106 - Replace one module with another
1108 After creation, this object should be added to the front of the sys.meta_path
1109 list (which may need to be created). The sys.path_importer_cache dictionary
1110 should also be cleared, to prevent loading any non-restricted modules.
1112 See PEP302 for more info on how this works:
1113 http://www.python.org/dev/peps/pep-0302/
1116 ENABLE_LOGGING
= False
1118 def log(self
, message
, *args
):
1119 """Logs an import-related message to stderr, with indentation based on
1120 current call-stack depth.
1123 message: Logging format string.
1124 args: Positional format parameters for the logging message.
1126 if HardenedModulesHook
.ENABLE_LOGGING
:
1127 indent
= self
._indent
_level
* ' '
1128 print >>sys
.stderr
, indent
+ (message
% args
)
1130 _WHITE_LIST_C_MODULES
= [
1144 '_Crypto_Cipher__AES',
1145 '_Crypto_Cipher__ARC2',
1146 '_Crypto_Cipher__ARC4',
1147 '_Crypto_Cipher__Blowfish',
1148 '_Crypto_Cipher__CAST',
1149 '_Crypto_Cipher__DES',
1150 '_Crypto_Cipher__DES3',
1151 '_Crypto_Cipher__XOR',
1152 '_Crypto_Hash__MD2',
1153 '_Crypto_Hash__MD4',
1154 '_Crypto_Hash__RIPEMD',
1155 '_Crypto_Hash__SHA256',
1211 __CRYPTO_CIPHER_ALLOWED_MODULES
= [
1221 _WHITE_LIST_PARTIAL_MODULES
= {
1222 'Crypto.Cipher.AES': __CRYPTO_CIPHER_ALLOWED_MODULES
,
1223 'Crypto.Cipher.ARC2': __CRYPTO_CIPHER_ALLOWED_MODULES
,
1224 'Crypto.Cipher.Blowfish': __CRYPTO_CIPHER_ALLOWED_MODULES
,
1225 'Crypto.Cipher.CAST': __CRYPTO_CIPHER_ALLOWED_MODULES
,
1226 'Crypto.Cipher.DES': __CRYPTO_CIPHER_ALLOWED_MODULES
,
1227 'Crypto.Cipher.DES3': __CRYPTO_CIPHER_ALLOWED_MODULES
,
1323 _MODULE_OVERRIDES
= {
1325 'setlocale': FakeSetLocale
,
1329 'access': FakeAccess
,
1330 'listdir': RestrictedPathFunction(os
.listdir
),
1332 'lstat': RestrictedPathFunction(os
.stat
),
1334 'readlink': FakeReadlink
,
1335 'remove': FakeUnlink
,
1336 'rename': FakeRename
,
1337 'stat': RestrictedPathFunction(os
.stat
),
1339 'unlink': FakeUnlink
,
1340 'urandom': FakeURandom
,
1345 'get_platform': FakeGetPlatform
,
1349 _ENABLED_FILE_TYPES
= (
1360 dummy_thread_module
=dummy_thread
,
1361 pickle_module
=pickle
):
1365 module_dict: Module dictionary to use for managing system modules.
1366 Should be sys.modules.
1367 imp_module, os_module, dummy_thread_module, pickle_module: References to
1368 modules that exist in the dev_appserver that must be used by this class
1369 in order to function, even if these modules have been unloaded from
1372 self
._module
_dict
= module_dict
1373 self
._imp
= imp_module
1374 self
._os
= os_module
1375 self
._dummy
_thread
= dummy_thread_module
1376 self
._pickle
= pickle
1377 self
._indent
_level
= 0
1380 def find_module(self
, fullname
, path
=None):
1382 if fullname
in ('cPickle', 'thread'):
1386 all_modules
= fullname
.split('.')
1388 for index
, current_module
in enumerate(all_modules
):
1389 current_module_fullname
= '.'.join(all_modules
[:index
+ 1])
1390 if (current_module_fullname
== fullname
and not
1391 self
.StubModuleExists(fullname
)):
1392 self
.FindModuleRestricted(current_module
,
1393 current_module_fullname
,
1396 if current_module_fullname
in self
._module
_dict
:
1397 module
= self
._module
_dict
[current_module_fullname
]
1399 module
= self
.FindAndLoadModule(current_module
,
1400 current_module_fullname
,
1403 if hasattr(module
, '__path__'):
1404 search_path
= module
.__path
__
1405 except CouldNotFindModuleError
:
1410 def StubModuleExists(self
, name
):
1411 """Check if the named module has a stub replacement."""
1412 if name
in sys
.builtin_module_names
:
1413 name
= 'py_%s' % name
1414 if name
in dist
.__all
__:
1418 def ImportStubModule(self
, name
):
1419 """Import the stub module replacement for the specified module."""
1420 if name
in sys
.builtin_module_names
:
1421 name
= 'py_%s' % name
1422 module
= __import__(dist
.__name
__, {}, {}, [name
])
1423 return getattr(module
, name
)
1426 def FixModule(self
, module
):
1427 """Prunes and overrides restricted module attributes.
1430 module: The module to prune. This should be a new module whose attributes
1431 reference back to the real module's __dict__ members.
1433 if module
.__name
__ in self
._WHITE
_LIST
_PARTIAL
_MODULES
:
1434 allowed_symbols
= self
._WHITE
_LIST
_PARTIAL
_MODULES
[module
.__name
__]
1435 for symbol
in set(module
.__dict
__) - set(allowed_symbols
):
1436 if not (symbol
.startswith('__') and symbol
.endswith('__')):
1437 del module
.__dict
__[symbol
]
1439 if module
.__name
__ in self
._MODULE
_OVERRIDES
:
1440 module
.__dict
__.update(self
._MODULE
_OVERRIDES
[module
.__name
__])
1443 def FindModuleRestricted(self
,
1447 """Locates a module while enforcing module import restrictions.
1450 submodule: The short name of the submodule (i.e., the last section of
1451 the fullname; for 'foo.bar' this would be 'bar').
1452 submodule_fullname: The fully qualified name of the module to find (e.g.,
1454 search_path: List of paths to search for to find this module. Should be
1455 None if the current sys.path should be used.
1458 Tuple (source_file, pathname, description) where:
1459 source_file: File-like object that contains the module; in the case
1460 of packages, this will be None, which implies to look at __init__.py.
1461 pathname: String containing the full path of the module on disk.
1462 description: Tuple returned by imp.find_module().
1463 However, in the case of an import using a path hook (e.g. a zipfile),
1464 source_file will be a PEP-302-style loader object, pathname will be None,
1465 and description will be a tuple filled with None values.
1468 ImportError exception if the requested module was found, but importing
1471 CouldNotFindModuleError exception if the request module could not even
1472 be found for import.
1474 if search_path
is None:
1475 search_path
= [None] + sys
.path
1476 for path_entry
in search_path
:
1477 result
= self
.FindPathHook(submodule
, submodule_fullname
, path_entry
)
1478 if result
is not None:
1479 source_file
, pathname
, description
= result
1480 if description
== (None, None, None):
1485 self
.log('Could not find module "%s"', submodule_fullname
)
1486 raise CouldNotFindModuleError()
1488 suffix
, mode
, file_type
= description
1490 if (file_type
not in (self
._imp
.C_BUILTIN
, self
._imp
.C_EXTENSION
) and
1491 not FakeFile
.IsFileAccessible(pathname
)):
1492 error_message
= 'Access to module file denied: %s' % pathname
1493 logging
.debug(error_message
)
1494 raise ImportError(error_message
)
1496 if (file_type
not in self
._ENABLED
_FILE
_TYPES
and
1497 submodule
not in self
._WHITE
_LIST
_C
_MODULES
):
1498 error_message
= ('Could not import "%s": Disallowed C-extension '
1499 'or built-in module' % submodule_fullname
)
1500 logging
.debug(error_message
)
1501 raise ImportError(error_message
)
1503 return source_file
, pathname
, description
1505 def FindPathHook(self
, submodule
, submodule_fullname
, path_entry
):
1506 """Helper for FindModuleRestricted to find a module in a sys.path entry.
1511 path_entry: A single sys.path entry, or None representing the builtins.
1514 Either None (if nothing was found), or a triple (source_file, path_name,
1515 description). See the doc string for FindModuleRestricted() for the
1516 meaning of the latter.
1518 if path_entry
is None:
1519 if submodule_fullname
in sys
.builtin_module_names
:
1521 result
= self
._imp
.find_module(submodule
)
1525 source_file
, pathname
, description
= result
1526 suffix
, mode
, file_type
= description
1527 if file_type
== self
._imp
.C_BUILTIN
:
1532 if path_entry
in sys
.path_importer_cache
:
1533 importer
= sys
.path_importer_cache
[path_entry
]
1536 for hook
in sys
.path_hooks
:
1538 importer
= hook(path_entry
)
1542 sys
.path_importer_cache
[path_entry
] = importer
1544 if importer
is None:
1546 return self
._imp
.find_module(submodule
, [path_entry
])
1550 loader
= importer
.find_module(submodule
)
1551 if loader
is not None:
1552 return (loader
, None, (None, None, None))
1557 def LoadModuleRestricted(self
,
1562 """Loads a module while enforcing module import restrictions.
1564 As a byproduct, the new module will be added to the module dictionary.
1567 submodule_fullname: The fully qualified name of the module to find (e.g.,
1569 source_file: File-like object that contains the module's source code,
1570 or a PEP-302-style loader object.
1571 pathname: String containing the full path of the module on disk.
1572 description: Tuple returned by imp.find_module(), or (None, None, None)
1573 in case source_file is a PEP-302-style loader object.
1579 ImportError exception of the specified module could not be loaded for
1582 if description
== (None, None, None):
1583 return source_file
.load_module(submodule_fullname
)
1587 return self
._imp
.load_module(submodule_fullname
,
1592 if submodule_fullname
in self
._module
_dict
:
1593 del self
._module
_dict
[submodule_fullname
]
1597 if source_file
is not None:
1601 def FindAndLoadModule(self
,
1605 """Finds and loads a module, loads it, and adds it to the module dictionary.
1608 submodule: Name of the module to import (e.g., baz).
1609 submodule_fullname: Full name of the module to import (e.g., foo.bar.baz).
1610 search_path: Path to use for searching for this submodule. For top-level
1611 modules this should be None; otherwise it should be the __path__
1612 attribute from the parent package.
1615 A new module instance that has been inserted into the module dictionary
1616 supplied to __init__.
1619 ImportError exception if the module could not be loaded for whatever
1620 reason (e.g., missing, not allowed).
1622 module
= self
._imp
.new_module(submodule_fullname
)
1624 if submodule_fullname
== 'thread':
1625 module
.__dict
__.update(self
._dummy
_thread
.__dict
__)
1626 module
.__name
__ = 'thread'
1627 elif submodule_fullname
== 'cPickle':
1628 module
.__dict
__.update(self
._pickle
.__dict
__)
1629 module
.__name
__ = 'cPickle'
1630 elif submodule_fullname
== 'os':
1631 module
.__dict
__.update(self
._os
.__dict
__)
1632 elif self
.StubModuleExists(submodule_fullname
):
1633 module
= self
.ImportStubModule(submodule_fullname
)
1635 source_file
, pathname
, description
= self
.FindModuleRestricted(submodule
, submodule_fullname
, search_path
)
1636 module
= self
.LoadModuleRestricted(submodule_fullname
,
1641 module
.__loader
__ = self
1642 self
.FixModule(module
)
1643 if submodule_fullname
not in self
._module
_dict
:
1644 self
._module
_dict
[submodule_fullname
] = module
1646 if submodule_fullname
== 'os':
1647 os_path_name
= module
.path
.__name
__
1648 os_path
= self
.FindAndLoadModule(os_path_name
, os_path_name
, search_path
)
1649 self
._module
_dict
['os.path'] = os_path
1650 module
.__dict
__['path'] = os_path
1655 def GetParentPackage(self
, fullname
):
1656 """Retrieves the parent package of a fully qualified module name.
1659 fullname: Full name of the module whose parent should be retrieved (e.g.,
1663 Module instance for the parent or None if there is no parent module.
1666 ImportError exception if the module's parent could not be found.
1668 all_modules
= fullname
.split('.')
1669 parent_module_fullname
= '.'.join(all_modules
[:-1])
1670 if parent_module_fullname
:
1671 if self
.find_module(fullname
) is None:
1672 raise ImportError('Could not find module %s' % fullname
)
1674 return self
._module
_dict
[parent_module_fullname
]
1678 def GetParentSearchPath(self
, fullname
):
1679 """Determines the search path of a module's parent package.
1682 fullname: Full name of the module to look up (e.g., foo.bar).
1685 Tuple (submodule, search_path) where:
1686 submodule: The last portion of the module name from fullname (e.g.,
1687 if fullname is foo.bar, then this is bar).
1688 search_path: List of paths that belong to the parent package's search
1689 path or None if there is no parent package.
1692 ImportError exception if the module or its parent could not be found.
1694 submodule
= GetSubmoduleName(fullname
)
1695 parent_package
= self
.GetParentPackage(fullname
)
1697 if parent_package
is not None and hasattr(parent_package
, '__path__'):
1698 search_path
= parent_package
.__path
__
1699 return submodule
, search_path
1702 def GetModuleInfo(self
, fullname
):
1703 """Determines the path on disk and the search path of a module or package.
1706 fullname: Full name of the module to look up (e.g., foo.bar).
1709 Tuple (pathname, search_path, submodule) where:
1710 pathname: String containing the full path of the module on disk,
1711 or None if the module wasn't loaded from disk (e.g. from a zipfile).
1712 search_path: List of paths that belong to the found package's search
1713 path or None if found module is not a package.
1714 submodule: The relative name of the submodule that's being imported.
1716 submodule
, search_path
= self
.GetParentSearchPath(fullname
)
1717 source_file
, pathname
, description
= self
.FindModuleRestricted(submodule
, fullname
, search_path
)
1718 suffix
, mode
, file_type
= description
1719 module_search_path
= None
1720 if file_type
== self
._imp
.PKG_DIRECTORY
:
1721 module_search_path
= [pathname
]
1722 pathname
= os
.path
.join(pathname
, '__init__%spy' % os
.extsep
)
1723 return pathname
, module_search_path
, submodule
1726 def load_module(self
, fullname
):
1728 all_modules
= fullname
.split('.')
1729 submodule
= all_modules
[-1]
1730 parent_module_fullname
= '.'.join(all_modules
[:-1])
1732 if parent_module_fullname
and parent_module_fullname
in self
._module
_dict
:
1733 parent_module
= self
._module
_dict
[parent_module_fullname
]
1734 if hasattr(parent_module
, '__path__'):
1735 search_path
= parent_module
.__path
__
1737 return self
.FindAndLoadModule(submodule
, fullname
, search_path
)
1740 def is_package(self
, fullname
):
1741 """See PEP 302 extensions."""
1742 submodule
, search_path
= self
.GetParentSearchPath(fullname
)
1743 source_file
, pathname
, description
= self
.FindModuleRestricted(submodule
, fullname
, search_path
)
1744 suffix
, mode
, file_type
= description
1745 if file_type
== self
._imp
.PKG_DIRECTORY
:
1750 def get_source(self
, fullname
):
1751 """See PEP 302 extensions."""
1752 full_path
, search_path
, submodule
= self
.GetModuleInfo(fullname
)
1753 if full_path
is None:
1755 source_file
= open(full_path
)
1757 return source_file
.read()
1762 def get_code(self
, fullname
):
1763 """See PEP 302 extensions."""
1764 full_path
, search_path
, submodule
= self
.GetModuleInfo(fullname
)
1765 if full_path
is None:
1767 source_file
= open(full_path
)
1769 source_code
= source_file
.read()
1773 source_code
= source_code
.replace('\r\n', '\n')
1774 if not source_code
.endswith('\n'):
1777 return compile(source_code
, full_path
, 'exec')
1780 def ModuleHasValidMainFunction(module
):
1781 """Determines if a module has a main function that takes no arguments.
1783 This includes functions that have arguments with defaults that are all
1784 assigned, thus requiring no additional arguments in order to be called.
1787 module: A types.ModuleType instance.
1790 True if the module has a valid, reusable main function; False otherwise.
1792 if hasattr(module
, 'main') and type(module
.main
) is types
.FunctionType
:
1793 arg_names
, var_args
, var_kwargs
, default_values
= inspect
.getargspec(module
.main
)
1794 if len(arg_names
) == 0:
1796 if default_values
is not None and len(arg_names
) == len(default_values
):
1801 def GetScriptModuleName(handler_path
):
1802 """Determines the fully-qualified Python module name of a script on disk.
1805 handler_path: CGI path stored in the application configuration (as a path
1806 like 'foo/bar/baz.py'). May contain $PYTHON_LIB references.
1809 String containing the corresponding module name (e.g., 'foo.bar.baz').
1811 if handler_path
.startswith(PYTHON_LIB_VAR
+ '/'):
1812 handler_path
= handler_path
[len(PYTHON_LIB_VAR
):]
1813 handler_path
= os
.path
.normpath(handler_path
)
1815 extension_index
= handler_path
.rfind('.py')
1816 if extension_index
!= -1:
1817 handler_path
= handler_path
[:extension_index
]
1818 module_fullname
= handler_path
.replace(os
.sep
, '.')
1819 module_fullname
= module_fullname
.strip('.')
1820 module_fullname
= re
.sub('\.+', '.', module_fullname
)
1822 if module_fullname
.endswith('.__init__'):
1823 module_fullname
= module_fullname
[:-len('.__init__')]
1825 return module_fullname
1828 def FindMissingInitFiles(cgi_path
, module_fullname
, isfile
=os
.path
.isfile
):
1829 """Determines which __init__.py files are missing from a module's parent
1833 cgi_path: Absolute path of the CGI module file on disk.
1834 module_fullname: Fully qualified Python module name used to import the
1838 List containing the paths to the missing __init__.py files.
1840 missing_init_files
= []
1842 if cgi_path
.endswith('.py'):
1843 module_base
= os
.path
.dirname(cgi_path
)
1845 module_base
= cgi_path
1847 depth_count
= module_fullname
.count('.')
1848 if cgi_path
.endswith('__init__.py') or not cgi_path
.endswith('.py'):
1851 for index
in xrange(depth_count
):
1852 current_init_file
= os
.path
.abspath(
1853 os
.path
.join(module_base
, '__init__.py'))
1855 if not isfile(current_init_file
):
1856 missing_init_files
.append(current_init_file
)
1858 module_base
= os
.path
.abspath(os
.path
.join(module_base
, os
.pardir
))
1860 return missing_init_files
1863 def LoadTargetModule(handler_path
,
1866 module_dict
=sys
.modules
):
1867 """Loads a target CGI script by importing it as a Python module.
1869 If the module for the target CGI script has already been loaded before,
1870 the new module will be loaded in its place using the same module object,
1871 possibly overwriting existing module attributes.
1874 handler_path: CGI path stored in the application configuration (as a path
1875 like 'foo/bar/baz.py'). Should not have $PYTHON_LIB references.
1876 cgi_path: Absolute path to the CGI script file on disk.
1877 import_hook: Instance of HardenedModulesHook to use for module loading.
1878 module_dict: Used for dependency injection.
1881 Tuple (module_fullname, script_module, module_code) where:
1882 module_fullname: Fully qualified module name used to import the script.
1883 script_module: The ModuleType object corresponding to the module_fullname.
1884 If the module has not already been loaded, this will be an empty
1886 module_code: Code object (returned by compile built-in) corresponding
1887 to the cgi_path to run. If the script_module was previously loaded
1888 and has a main() function that can be reused, this will be None.
1890 module_fullname
= GetScriptModuleName(handler_path
)
1891 script_module
= module_dict
.get(module_fullname
)
1893 if script_module
!= None and ModuleHasValidMainFunction(script_module
):
1894 logging
.debug('Reusing main() function of module "%s"', module_fullname
)
1896 if script_module
is None:
1897 script_module
= imp
.new_module(module_fullname
)
1898 script_module
.__loader
__ = import_hook
1901 module_code
= import_hook
.get_code(module_fullname
)
1902 full_path
, search_path
, submodule
= import_hook
.GetModuleInfo(module_fullname
)
1903 script_module
.__file
__ = full_path
1904 if search_path
is not None:
1905 script_module
.__path
__ = search_path
1907 exc_type
, exc_value
, exc_tb
= sys
.exc_info()
1908 import_error_message
= str(exc_type
)
1910 import_error_message
+= ': ' + str(exc_value
)
1912 logging
.exception('Encountered error loading module "%s": %s',
1913 module_fullname
, import_error_message
)
1914 missing_inits
= FindMissingInitFiles(cgi_path
, module_fullname
)
1916 logging
.warning('Missing package initialization files: %s',
1917 ', '.join(missing_inits
))
1919 logging
.error('Parent package initialization files are present, '
1920 'but must be broken')
1922 independent_load_successful
= True
1924 if not os
.path
.isfile(cgi_path
):
1925 independent_load_successful
= False
1928 source_file
= open(cgi_path
)
1930 module_code
= compile(source_file
.read(), cgi_path
, 'exec')
1931 script_module
.__file
__ = cgi_path
1936 independent_load_successful
= False
1938 if not independent_load_successful
:
1939 raise exc_type
, exc_value
, exc_tb
1941 module_dict
[module_fullname
] = script_module
1943 return module_fullname
, script_module
, module_code
1946 def ExecuteOrImportScript(handler_path
, cgi_path
, import_hook
):
1947 """Executes a CGI script by importing it as a new module; possibly reuses
1948 the module's main() function if it is defined and takes no arguments.
1950 Basic technique lifted from PEP 338 and Python2.5's runpy module. See:
1951 http://www.python.org/dev/peps/pep-0338/
1953 See the section entitled "Import Statements and the Main Module" to understand
1954 why a module named '__main__' cannot do relative imports. To get around this,
1955 the requested module's path could be added to sys.path on each request.
1958 handler_path: CGI path stored in the application configuration (as a path
1959 like 'foo/bar/baz.py'). Should not have $PYTHON_LIB references.
1960 cgi_path: Absolute path to the CGI script file on disk.
1961 import_hook: Instance of HardenedModulesHook to use for module loading.
1964 True if the response code had an error status (e.g., 404), or False if it
1968 Any kind of exception that could have been raised when loading the target
1969 module, running a target script, or executing the application code itself.
1971 module_fullname
, script_module
, module_code
= LoadTargetModule(
1972 handler_path
, cgi_path
, import_hook
)
1973 script_module
.__name
__ = '__main__'
1974 sys
.modules
['__main__'] = script_module
1977 exec module_code
in script_module
.__dict
__
1979 script_module
.main()
1984 headers
= mimetools
.Message(sys
.stdout
)
1986 sys
.stdout
.seek(0, 2)
1987 status_header
= headers
.get('status')
1988 error_response
= False
1991 status_code
= int(status_header
.split(' ', 1)[0])
1992 error_response
= status_code
>= 400
1994 error_response
= True
1996 if not error_response
:
1998 parent_package
= import_hook
.GetParentPackage(module_fullname
)
2000 parent_package
= None
2002 if parent_package
is not None:
2003 submodule
= GetSubmoduleName(module_fullname
)
2004 setattr(parent_package
, submodule
, script_module
)
2006 return error_response
2008 script_module
.__name
__ = module_fullname
2011 def ExecuteCGI(root_path
,
2018 exec_script
=ExecuteOrImportScript
):
2019 """Executes Python file in this process as if it were a CGI.
2021 Does not return an HTTP response line. CGIs should output headers followed by
2024 The modules in sys.modules should be the same before and after the CGI is
2025 executed, with the specific exception of encodings-related modules, which
2026 cannot be reloaded and thus must always stay in sys.modules.
2029 root_path: Path to the root of the application.
2030 handler_path: CGI path stored in the application configuration (as a path
2031 like 'foo/bar/baz.py'). May contain $PYTHON_LIB references.
2032 cgi_path: Absolute path to the CGI script file on disk.
2033 env: Dictionary of environment variables to use for the execution.
2034 infile: File-like object to read HTTP request input data from.
2035 outfile: FIle-like object to write HTTP response data to.
2036 module_dict: Dictionary in which application-loaded modules should be
2037 preserved between requests. This removes the need to reload modules that
2038 are reused between requests, significantly increasing load performance.
2039 This dictionary must be separate from the sys.modules dictionary.
2040 exec_script: Used for dependency injection.
2042 old_module_dict
= sys
.modules
.copy()
2043 old_builtin
= __builtin__
.__dict
__.copy()
2045 old_stdin
= sys
.stdin
2046 old_stdout
= sys
.stdout
2047 old_env
= os
.environ
.copy()
2048 old_cwd
= os
.getcwd()
2049 old_file_type
= types
.FileType
2050 reset_modules
= False
2053 ClearAllButEncodingsModules(sys
.modules
)
2054 sys
.modules
.update(module_dict
)
2055 sys
.argv
= [cgi_path
]
2057 sys
.stdout
= outfile
2059 os
.environ
.update(env
)
2060 before_path
= sys
.path
[:]
2061 cgi_dir
= os
.path
.normpath(os
.path
.dirname(cgi_path
))
2062 root_path
= os
.path
.normpath(os
.path
.abspath(root_path
))
2063 if cgi_dir
.startswith(root_path
+ os
.sep
):
2068 hook
= HardenedModulesHook(sys
.modules
)
2069 sys
.meta_path
= [hook
]
2070 if hasattr(sys
, 'path_importer_cache'):
2071 sys
.path_importer_cache
.clear()
2073 __builtin__
.file = FakeFile
2074 __builtin__
.open = FakeFile
2075 types
.FileType
= FakeFile
2077 __builtin__
.buffer = NotImplementedFakeClass
2079 logging
.debug('Executing CGI with env:\n%s', pprint
.pformat(env
))
2081 reset_modules
= exec_script(handler_path
, cgi_path
, hook
)
2082 except SystemExit, e
:
2083 logging
.debug('CGI exited with status: %s', e
)
2085 reset_modules
= True
2090 sys
.path_importer_cache
.clear()
2092 _ClearTemplateCache(sys
.modules
)
2094 module_dict
.update(sys
.modules
)
2095 ClearAllButEncodingsModules(sys
.modules
)
2096 sys
.modules
.update(old_module_dict
)
2098 __builtin__
.__dict
__.update(old_builtin
)
2100 sys
.stdin
= old_stdin
2101 sys
.stdout
= old_stdout
2103 sys
.path
[:] = before_path
2106 os
.environ
.update(old_env
)
2109 types
.FileType
= old_file_type
2112 class CGIDispatcher(URLDispatcher
):
2113 """Dispatcher that executes Python CGI scripts."""
2119 setup_env
=SetupEnvironment
,
2120 exec_cgi
=ExecuteCGI
,
2121 create_logging_handler
=ApplicationLoggingHandler
):
2125 module_dict: Dictionary in which application-loaded modules should be
2126 preserved between requests. This dictionary must be separate from the
2127 sys.modules dictionary.
2128 path_adjuster: Instance of PathAdjuster to use for finding absolute
2129 paths of CGI files on disk.
2130 setup_env, exec_cgi, create_logging_handler: Used for dependency
2133 self
._module
_dict
= module_dict
2134 self
._root
_path
= root_path
2135 self
._path
_adjuster
= path_adjuster
2136 self
._setup
_env
= setup_env
2137 self
._exec
_cgi
= exec_cgi
2138 self
._create
_logging
_handler
= create_logging_handler
2146 base_env_dict
=None):
2147 """Dispatches the Python CGI."""
2148 handler
= self
._create
_logging
_handler
()
2149 logging
.getLogger().addHandler(handler
)
2150 before_level
= logging
.root
.level
2154 env
.update(base_env_dict
)
2155 cgi_path
= self
._path
_adjuster
.AdjustPath(path
)
2156 env
.update(self
._setup
_env
(cgi_path
, relative_url
, headers
))
2157 self
._exec
_cgi
(self
._root
_path
,
2164 handler
.AddDebuggingConsole(relative_url
, env
, outfile
)
2166 logging
.root
.level
= before_level
2167 logging
.getLogger().removeHandler(handler
)
2170 """Returns a string representation of this dispatcher."""
2171 return 'CGI dispatcher'
2174 class LocalCGIDispatcher(CGIDispatcher
):
2175 """Dispatcher that executes local functions like they're CGIs.
2177 The contents of sys.modules will be preserved for local CGIs running this
2178 dispatcher, but module hardening will still occur for any new imports. Thus,
2179 be sure that any local CGIs have loaded all of their dependent modules
2180 _before_ they are executed.
2183 def __init__(self
, module_dict
, path_adjuster
, cgi_func
):
2187 module_dict: Passed to CGIDispatcher.
2188 path_adjuster: Passed to CGIDispatcher.
2189 cgi_func: Callable function taking no parameters that should be
2190 executed in a CGI environment in the current process.
2192 self
._cgi
_func
= cgi_func
2194 def curried_exec_script(*args
, **kwargs
):
2198 def curried_exec_cgi(*args
, **kwargs
):
2199 kwargs
['exec_script'] = curried_exec_script
2200 return ExecuteCGI(*args
, **kwargs
)
2202 CGIDispatcher
.__init
__(self
,
2206 exec_cgi
=curried_exec_cgi
)
2208 def Dispatch(self
, *args
, **kwargs
):
2209 """Preserves sys.modules for CGIDispatcher.Dispatch."""
2210 self
._module
_dict
.update(sys
.modules
)
2211 CGIDispatcher
.Dispatch(self
, *args
, **kwargs
)
2214 """Returns a string representation of this dispatcher."""
2215 return 'Local CGI dispatcher for %s' % self
._cgi
_func
2218 class PathAdjuster(object):
2219 """Adjusts application file paths to paths relative to the application or
2220 external library directories."""
2222 def __init__(self
, root_path
):
2226 root_path: Path to the root of the application running on the server.
2228 self
._root
_path
= os
.path
.abspath(root_path
)
2230 def AdjustPath(self
, path
):
2231 """Adjusts application file path to paths relative to the application or
2232 external library directories.
2234 Handler paths that start with $PYTHON_LIB will be converted to paths
2235 relative to the google directory.
2238 path: File path that should be adjusted.
2243 if path
.startswith(PYTHON_LIB_VAR
):
2244 path
= os
.path
.join(os
.path
.dirname(os
.path
.dirname(google
.__file
__)),
2245 path
[len(PYTHON_LIB_VAR
) + 1:])
2247 path
= os
.path
.join(self
._root
_path
, path
)
2252 class StaticFileConfigMatcher(object):
2253 """Keeps track of file/directory specific application configuration.
2256 - Computes mime type based on URLMap and file extension.
2257 - Decides on cache expiration time based on URLMap and default expiration.
2259 To determine the mime type, we first see if there is any mime-type property
2260 on each URLMap entry. If non is specified, we use the mimetypes module to
2261 guess the mime type from the file path extension, and use
2262 application/octet-stream if we can't find the mimetype.
2268 default_expiration
):
2272 url_map_list: List of appinfo.URLMap objects.
2273 If empty or None, then we always use the mime type chosen by the
2275 path_adjuster: PathAdjuster object used to adjust application file paths.
2276 default_expiration: String describing default expiration time for browser
2277 based caching of static files. If set to None this disallows any
2278 browser caching of static content.
2280 if default_expiration
is not None:
2281 self
._default
_expiration
= appinfo
.ParseExpiration(default_expiration
)
2283 self
._default
_expiration
= None
2288 for entry
in url_map_list
:
2289 handler_type
= entry
.GetHandlerType()
2290 if handler_type
not in (appinfo
.STATIC_FILES
, appinfo
.STATIC_DIR
):
2293 if handler_type
== appinfo
.STATIC_FILES
:
2294 regex
= entry
.upload
+ '$'
2296 path
= entry
.static_dir
2299 regex
= re
.escape(path
+ os
.path
.sep
) + r
'(.*)'
2302 path_re
= re
.compile(regex
)
2304 raise InvalidAppConfigError('regex %s does not compile: %s' %
2307 if self
._default
_expiration
is None:
2309 elif entry
.expiration
is None:
2310 expiration
= self
._default
_expiration
2312 expiration
= appinfo
.ParseExpiration(entry
.expiration
)
2314 self
._patterns
.append((path_re
, entry
.mime_type
, expiration
))
2316 def IsStaticFile(self
, path
):
2317 """Tests if the given path points to a "static" file.
2320 path: String containing the file's path relative to the app.
2323 Boolean, True if the file was configured to be static.
2325 for (path_re
, _
, _
) in self
._patterns
:
2326 if path_re
.match(path
):
2330 def GetMimeType(self
, path
):
2331 """Returns the mime type that we should use when serving the specified file.
2334 path: String containing the file's path relative to the app.
2337 String containing the mime type to use. Will be 'application/octet-stream'
2338 if we have no idea what it should be.
2340 for (path_re
, mime_type
, expiration
) in self
._patterns
:
2341 if mime_type
is not None:
2342 the_match
= path_re
.match(path
)
2346 filename
, extension
= os
.path
.splitext(path
)
2347 return mimetypes
.types_map
.get(extension
, 'application/octet-stream')
2349 def GetExpiration(self
, path
):
2350 """Returns the cache expiration duration to be users for the given file.
2353 path: String containing the file's path relative to the app.
2356 Integer number of seconds to be used for browser cache expiration time.
2358 for (path_re
, mime_type
, expiration
) in self
._patterns
:
2359 the_match
= path_re
.match(path
)
2363 return self
._default
_expiration
or 0
2367 def ReadDataFile(data_path
, openfile
=file):
2368 """Reads a file on disk, returning a corresponding HTTP status and data.
2371 data_path: Path to the file on disk to read.
2372 openfile: Used for dependency injection.
2375 Tuple (status, data) where status is an HTTP response code, and data is
2376 the data read; will be an empty string if an error occurred or the
2379 status
= httplib
.INTERNAL_SERVER_ERROR
2383 data_file
= openfile(data_path
, 'rb')
2385 data
= data_file
.read()
2389 except (OSError, IOError), e
:
2390 logging
.error('Error encountered reading file "%s":\n%s', data_path
, e
)
2391 if e
.errno
in FILE_MISSING_EXCEPTIONS
:
2392 status
= httplib
.NOT_FOUND
2394 status
= httplib
.FORBIDDEN
2399 class FileDispatcher(URLDispatcher
):
2400 """Dispatcher that reads data files from disk."""
2404 static_file_config_matcher
,
2405 read_data_file
=ReadDataFile
):
2409 path_adjuster: Instance of PathAdjuster to use for finding absolute
2410 paths of data files on disk.
2411 static_file_config_matcher: StaticFileConfigMatcher object.
2412 read_data_file: Used for dependency injection.
2414 self
._path
_adjuster
= path_adjuster
2415 self
._static
_file
_config
_matcher
= static_file_config_matcher
2416 self
._read
_data
_file
= read_data_file
2424 base_env_dict
=None):
2425 """Reads the file and returns the response status and data."""
2426 full_path
= self
._path
_adjuster
.AdjustPath(path
)
2427 status
, data
= self
._read
_data
_file
(full_path
)
2428 content_type
= self
._static
_file
_config
_matcher
.GetMimeType(path
)
2429 expiration
= self
._static
_file
_config
_matcher
.GetExpiration(path
)
2431 outfile
.write('Status: %d\r\n' % status
)
2432 outfile
.write('Content-type: %s\r\n' % content_type
)
2434 outfile
.write('Expires: %s\r\n'
2435 % email
.Utils
.formatdate(time
.time() + expiration
,
2437 outfile
.write('Cache-Control: public, max-age=%i\r\n' % expiration
)
2438 outfile
.write('\r\n')
2442 """Returns a string representation of this dispatcher."""
2443 return 'File dispatcher'
2446 _IGNORE_RESPONSE_HEADERS
= frozenset([
2447 'content-encoding', 'accept-encoding', 'transfer-encoding',
2452 def IgnoreHeadersRewriter(status_code
, status_message
, headers
, body
):
2453 """Ignore specific response headers.
2455 Certain response headers cannot be modified by an Application. For a
2456 complete list of these headers please see:
2458 http://code.google.com/appengine/docs/webapp/responseclass.html#Disallowed_HTTP_Response_Headers
2460 This rewriter simply removes those headers.
2462 for h
in _IGNORE_RESPONSE_HEADERS
:
2466 return status_code
, status_message
, headers
, body
2469 def ParseStatusRewriter(status_code
, status_message
, headers
, body
):
2470 """Parse status header, if it exists.
2472 Handles the server-side 'status' header, which instructs the server to change
2473 the HTTP response code accordingly. Handles the 'location' header, which
2474 issues an HTTP 302 redirect to the client. Also corrects the 'content-length'
2475 header to reflect actual content length in case extra information has been
2476 appended to the response body.
2478 If the 'status' header supplied by the client is invalid, this method will
2479 set the response to a 500 with an error message as content.
2481 location_value
= headers
.getheader('location')
2482 status_value
= headers
.getheader('status')
2484 response_status
= status_value
2485 del headers
['status']
2486 elif location_value
:
2487 response_status
= '%d Redirecting' % httplib
.FOUND
2489 return status_code
, status_message
, headers
, body
2491 status_parts
= response_status
.split(' ', 1)
2492 status_code
, status_message
= (status_parts
+ [''])[:2]
2494 status_code
= int(status_code
)
2497 body
= cStringIO
.StringIO('Error: Invalid "status" header value returned.')
2499 return status_code
, status_message
, headers
, body
2502 def CacheRewriter(status_code
, status_message
, headers
, body
):
2503 """Update the cache header."""
2504 if not 'Cache-Control' in headers
:
2505 headers
['Cache-Control'] = 'no-cache'
2506 return status_code
, status_message
, headers
, body
2509 def ContentLengthRewriter(status_code
, status_message
, headers
, body
):
2510 """Rewrite the Content-Length header.
2512 Even though Content-Length is not a user modifiable header, App Engine
2513 sends a correct Content-Length to the user based on the actual response.
2515 current_position
= body
.tell()
2518 headers
['Content-Length'] = str(body
.tell() - current_position
)
2519 body
.seek(current_position
)
2520 return status_code
, status_message
, headers
, body
2523 def CreateResponseRewritersChain():
2524 """Create the default response rewriter chain.
2526 A response rewriter is the a function that gets a final chance to change part
2527 of the dev_appservers response. A rewriter is not like a dispatcher in that
2528 it is called after every request has been handled by the dispatchers
2529 regardless of which dispatcher was used.
2531 The order in which rewriters are registered will be the order in which they
2532 are used to rewrite the response. Modifications from earlier rewriters
2533 are used as input to later rewriters.
2535 A response rewriter is a function that can rewrite the request in any way.
2536 Thefunction can returned modified values or the original values it was
2539 A rewriter function has the following parameters and return values:
2542 status_code: Status code of response from dev_appserver or previous
2544 status_message: Text corresponding to status code.
2545 headers: mimetools.Message instance with parsed headers. NOTE: These
2546 headers can contain its own 'status' field, but the default
2547 dev_appserver implementation will remove this. Future rewriters
2548 should avoid re-introducing the status field and return new codes
2550 body: File object containing the body of the response. This position of
2551 this file may not be at the start of the file. Any content before the
2552 files position is considered not to be part of the final body.
2555 status_code: Rewritten status code or original.
2556 status_message: Rewritter message or original.
2557 headers: Rewritten/modified headers or original.
2558 body: Rewritten/modified body or original.
2561 List of response rewriters.
2563 return [IgnoreHeadersRewriter
,
2564 ParseStatusRewriter
,
2566 ContentLengthRewriter
,
2570 def RewriteResponse(response_file
, response_rewriters
=None):
2571 """Allows final rewrite of dev_appserver response.
2573 This function receives the unparsed HTTP response from the application
2574 or internal handler, parses out the basic structure and feeds that structure
2575 in to a chain of response rewriters.
2577 It also makes sure the final HTTP headers are properly terminated.
2579 For more about response rewriters, please see documentation for
2580 CreateResponeRewritersChain.
2583 response_file: File-like object containing the full HTTP response including
2584 the response code, all headers, and the request body.
2585 response_rewriters: A list of response rewriters. If none is provided it
2586 will create a new chain using CreateResponseRewritersChain.
2589 Tuple (status_code, status_message, header, body) where:
2590 status_code: Integer HTTP response status (e.g., 200, 302, 404, 500)
2591 status_message: String containing an informational message about the
2592 response code, possibly derived from the 'status' header, if supplied.
2593 header: String containing the HTTP headers of the response, without
2594 a trailing new-line (CRLF).
2595 body: String containing the body of the response.
2597 if response_rewriters
is None:
2598 response_rewriters
= CreateResponseRewritersChain()
2601 status_message
= 'Good to go'
2602 headers
= mimetools
.Message(response_file
)
2604 for response_rewriter
in response_rewriters
:
2605 status_code
, status_message
, headers
, response_file
= response_rewriter(
2612 for header
in headers
.headers
:
2613 header
= header
.rstrip('\n')
2614 header
= header
.rstrip('\r')
2615 header_list
.append(header
)
2617 header_data
= '\r\n'.join(header_list
) + '\r\n'
2618 return status_code
, status_message
, header_data
, response_file
.read()
2621 class ModuleManager(object):
2622 """Manages loaded modules in the runtime.
2624 Responsible for monitoring and reporting about file modification times.
2625 Modules can be loaded from source or precompiled byte-code files. When a
2626 file has source code, the ModuleManager monitors the modification time of
2627 the source file even if the module itself is loaded from byte-code.
2630 def __init__(self
, modules
):
2634 modules: Dictionary containing monitored modules.
2636 self
._modules
= modules
2637 self
._default
_modules
= self
._modules
.copy()
2638 self
._save
_path
_hooks
= sys
.path_hooks
[:]
2639 self
._modification
_times
= {}
2642 def GetModuleFile(module
, is_file
=os
.path
.isfile
):
2643 """Helper method to try to determine modules source file.
2646 module: Module object to get file for.
2647 is_file: Function used to determine if a given path is a file.
2650 Path of the module's corresponding Python source file if it exists, or
2651 just the module's compiled Python file. If the module has an invalid
2652 __file__ attribute, None will be returned.
2654 module_file
= getattr(module
, '__file__', None)
2655 if module_file
is None:
2658 source_file
= module_file
[:module_file
.rfind('py') + 2]
2660 if is_file(source_file
):
2662 return module
.__file
__
2664 def AreModuleFilesModified(self
):
2665 """Determines if any monitored files have been modified.
2668 True if one or more files have been modified, False otherwise.
2670 for name
, (mtime
, fname
) in self
._modification
_times
.iteritems():
2671 if name
not in self
._modules
:
2674 module
= self
._modules
[name
]
2676 if not os
.path
.isfile(fname
):
2679 if mtime
!= os
.path
.getmtime(fname
):
2684 def UpdateModuleFileModificationTimes(self
):
2685 """Records the current modification times of all monitored modules.
2687 self
._modification
_times
.clear()
2688 for name
, module
in self
._modules
.items():
2689 if not isinstance(module
, types
.ModuleType
):
2691 module_file
= self
.GetModuleFile(module
)
2695 self
._modification
_times
[name
] = (os
.path
.getmtime(module_file
),
2698 if e
.errno
not in FILE_MISSING_EXCEPTIONS
:
2701 def ResetModules(self
):
2702 """Clear modules so that when request is run they are reloaded."""
2703 self
._modules
.clear()
2704 self
._modules
.update(self
._default
_modules
)
2705 sys
.path_hooks
[:] = self
._save
_path
_hooks
2708 def _ClearTemplateCache(module_dict
=sys
.modules
):
2709 """Clear template cache in webapp.template module.
2711 Attempts to load template module. Ignores failure. If module loads, the
2712 template cache is cleared.
2714 template_module
= module_dict
.get('google.appengine.ext.webapp.template')
2715 if template_module
is not None:
2716 template_module
.template_cache
.clear()
2719 def CreateRequestHandler(root_path
,
2721 require_indexes
=False,
2722 static_caching
=True):
2723 """Creates a new BaseHTTPRequestHandler sub-class for use with the Python
2724 BaseHTTPServer module's HTTP server.
2726 Python's built-in HTTP server does not support passing context information
2727 along to instances of its request handlers. This function gets around that
2728 by creating a sub-class of the handler in a closure that has access to
2729 this context information.
2732 root_path: Path to the root of the application running on the server.
2733 login_url: Relative URL which should be used for handling user logins.
2734 require_indexes: True if index.yaml is read-only gospel; default False.
2735 static_caching: True if browser caching of static files should be allowed.
2738 Sub-class of BaseHTTPRequestHandler.
2740 application_module_dict
= SetupSharedModules(sys
.modules
)
2743 index_yaml_updater
= None
2745 index_yaml_updater
= dev_appserver_index
.IndexYamlUpdater(root_path
)
2747 application_config_cache
= AppConfigCache()
2749 class DevAppServerRequestHandler(BaseHTTPServer
.BaseHTTPRequestHandler
):
2750 """Dispatches URLs using patterns from a URLMatcher, which is created by
2751 loading an application's configuration file. Executes CGI scripts in the
2752 local process so the scripts can use mock versions of APIs.
2754 HTTP requests that correctly specify a user info cookie
2755 (dev_appserver_login.COOKIE_NAME) will have the 'USER_EMAIL' environment
2756 variable set accordingly. If the user is also an admin, the
2757 'USER_IS_ADMIN' variable will exist and be set to '1'. If the user is not
2758 logged in, 'USER_EMAIL' will be set to the empty string.
2760 On each request, raises an InvalidAppConfigError exception if the
2761 application configuration file in the directory specified by the root_path
2762 argument is invalid.
2764 server_version
= 'Development/1.0'
2766 module_dict
= application_module_dict
2767 module_manager
= ModuleManager(application_module_dict
)
2769 config_cache
= application_config_cache
2771 rewriter_chain
= CreateResponseRewritersChain()
2773 def __init__(self
, *args
, **kwargs
):
2777 args, kwargs: Positional and keyword arguments passed to the constructor
2780 BaseHTTPServer
.BaseHTTPRequestHandler
.__init
__(self
, *args
, **kwargs
)
2782 def version_string(self
):
2783 """Returns server's version string used for Server HTTP header"""
2784 return self
.server_version
2787 """Handle GET requests."""
2788 self
._HandleRequest
()
2791 """Handles POST requests."""
2792 self
._HandleRequest
()
2795 """Handle PUT requests."""
2796 self
._HandleRequest
()
2799 """Handle HEAD requests."""
2800 self
._HandleRequest
()
2802 def do_OPTIONS(self
):
2803 """Handles OPTIONS requests."""
2804 self
._HandleRequest
()
2806 def do_DELETE(self
):
2807 """Handle DELETE requests."""
2808 self
._HandleRequest
()
2811 """Handles TRACE requests."""
2812 self
._HandleRequest
()
2814 def _HandleRequest(self
):
2815 """Handles any type of request and prints exceptions if they occur."""
2816 server_name
= self
.headers
.get('host') or self
.server
.server_name
2817 server_name
= server_name
.split(':', 1)[0]
2820 'REQUEST_METHOD': self
.command
,
2821 'REMOTE_ADDR': self
.client_address
[0],
2822 'SERVER_SOFTWARE': self
.server_version
,
2823 'SERVER_NAME': server_name
,
2824 'SERVER_PROTOCOL': self
.protocol_version
,
2825 'SERVER_PORT': str(self
.server
.server_port
),
2828 full_url
= GetFullURL(server_name
, self
.server
.server_port
, self
.path
)
2829 if len(full_url
) > MAX_URL_LENGTH
:
2830 msg
= 'Requested URI too long: %s' % full_url
2832 self
.send_response(httplib
.REQUEST_URI_TOO_LONG
, msg
)
2835 tbhandler
= cgitb
.Hook(file=self
.wfile
).handle
2837 if self
.module_manager
.AreModuleFilesModified():
2838 self
.module_manager
.ResetModules()
2840 implicit_matcher
= CreateImplicitMatcher(self
.module_dict
,
2843 config
, explicit_matcher
= LoadAppConfig(root_path
, self
.module_dict
,
2844 cache
=self
.config_cache
,
2845 static_caching
=static_caching
)
2846 if config
.api_version
!= API_VERSION
:
2847 logging
.error("API versions cannot be switched dynamically: %r != %r"
2848 % (config
.api_version
, API_VERSION
))
2850 env_dict
['CURRENT_VERSION_ID'] = config
.version
+ ".1"
2851 env_dict
['APPLICATION_ID'] = config
.application
2852 dispatcher
= MatcherDispatcher(login_url
,
2853 [implicit_matcher
, explicit_matcher
])
2856 dev_appserver_index
.SetupIndexes(config
.application
, root_path
)
2858 infile
= cStringIO
.StringIO(self
.rfile
.read(
2859 int(self
.headers
.get('content-length', 0))))
2861 request_size
= len(infile
.getvalue())
2862 if request_size
> MAX_REQUEST_SIZE
:
2863 msg
= ('HTTP request was too large: %d. The limit is: %d.'
2864 % (request_size
, MAX_REQUEST_SIZE
))
2866 self
.send_response(httplib
.REQUEST_ENTITY_TOO_LARGE
, msg
)
2869 outfile
= cStringIO
.StringIO()
2871 dispatcher
.Dispatch(self
.path
,
2876 base_env_dict
=env_dict
)
2878 self
.module_manager
.UpdateModuleFileModificationTimes()
2883 status_code
, status_message
, header_data
, body
= RewriteResponse(outfile
, self
.rewriter_chain
)
2885 runtime_response_size
= len(outfile
.getvalue())
2886 if runtime_response_size
> MAX_RUNTIME_RESPONSE_SIZE
:
2888 status_message
= 'Forbidden'
2890 for header
in header_data
.split('\n'):
2891 if not header
.lower().startswith('content-length'):
2892 new_headers
.append(header
)
2893 header_data
= '\n'.join(new_headers
)
2894 body
= ('HTTP response was too large: %d. The limit is: %d.'
2895 % (runtime_response_size
, MAX_RUNTIME_RESPONSE_SIZE
))
2897 except yaml_errors
.EventListenerError
, e
:
2898 title
= 'Fatal error when loading application configuration'
2899 msg
= '%s:\n%s' % (title
, str(e
))
2901 self
.send_response(httplib
.INTERNAL_SERVER_ERROR
, title
)
2902 self
.wfile
.write('Content-Type: text/html\n\n')
2903 self
.wfile
.write('<pre>%s</pre>' % cgi
.escape(msg
))
2905 msg
= 'Exception encountered handling request'
2906 logging
.exception(msg
)
2907 self
.send_response(httplib
.INTERNAL_SERVER_ERROR
, msg
)
2911 self
.send_response(status_code
, status_message
)
2912 self
.wfile
.write(header_data
)
2913 self
.wfile
.write('\r\n')
2914 if self
.command
!= 'HEAD':
2915 self
.wfile
.write(body
)
2917 logging
.warning('Dropping unexpected body in response '
2919 except (IOError, OSError), e
:
2920 if e
.errno
!= errno
.EPIPE
:
2922 except socket
.error
, e
:
2923 if len(e
.args
) >= 1 and e
.args
[0] != errno
.EPIPE
:
2926 if index_yaml_updater
is not None:
2927 index_yaml_updater
.UpdateIndexYaml()
2929 def log_error(self
, format
, *args
):
2930 """Redirect error messages through the logging module."""
2931 logging
.error(format
, *args
)
2933 def log_message(self
, format
, *args
):
2934 """Redirect log messages through the logging module."""
2935 logging
.info(format
, *args
)
2937 return DevAppServerRequestHandler
2940 def ReadAppConfig(appinfo_path
, parse_app_config
=appinfo
.LoadSingleAppInfo
):
2941 """Reads app.yaml file and returns its app id and list of URLMap instances.
2944 appinfo_path: String containing the path to the app.yaml file.
2945 parse_app_config: Used for dependency injection.
2948 AppInfoExternal instance.
2951 If the config file could not be read or the config does not contain any
2952 URLMap instances, this function will raise an InvalidAppConfigError
2956 appinfo_file
= file(appinfo_path
, 'r')
2958 raise InvalidAppConfigError(
2959 'Application configuration could not be read from "%s"' % appinfo_path
)
2961 return parse_app_config(appinfo_file
)
2963 appinfo_file
.close()
2966 def CreateURLMatcherFromMaps(root_path
,
2970 create_url_matcher
=URLMatcher
,
2971 create_cgi_dispatcher
=CGIDispatcher
,
2972 create_file_dispatcher
=FileDispatcher
,
2973 create_path_adjuster
=PathAdjuster
,
2974 normpath
=os
.path
.normpath
):
2975 """Creates a URLMatcher instance from URLMap.
2977 Creates all of the correct URLDispatcher instances to handle the various
2978 content types in the application configuration.
2981 root_path: Path to the root of the application running on the server.
2982 url_map_list: List of appinfo.URLMap objects to initialize this
2983 matcher with. Can be an empty list if you would like to add patterns
2985 module_dict: Dictionary in which application-loaded modules should be
2986 preserved between requests. This dictionary must be separate from the
2987 sys.modules dictionary.
2988 default_expiration: String describing default expiration time for browser
2989 based caching of static files. If set to None this disallows any
2990 browser caching of static content.
2991 create_url_matcher, create_cgi_dispatcher, create_file_dispatcher,
2992 create_path_adjuster: Used for dependency injection.
2995 Instance of URLMatcher with the supplied URLMap objects properly loaded.
2997 url_matcher
= create_url_matcher()
2998 path_adjuster
= create_path_adjuster(root_path
)
2999 cgi_dispatcher
= create_cgi_dispatcher(module_dict
, root_path
, path_adjuster
)
3000 static_file_config_matcher
= StaticFileConfigMatcher(url_map_list
,
3003 file_dispatcher
= create_file_dispatcher(path_adjuster
,
3004 static_file_config_matcher
)
3006 FakeFile
.SetStaticFileConfigMatcher(static_file_config_matcher
)
3008 for url_map
in url_map_list
:
3009 admin_only
= url_map
.login
== appinfo
.LOGIN_ADMIN
3010 requires_login
= url_map
.login
== appinfo
.LOGIN_REQUIRED
or admin_only
3012 handler_type
= url_map
.GetHandlerType()
3013 if handler_type
== appinfo
.HANDLER_SCRIPT
:
3014 dispatcher
= cgi_dispatcher
3015 elif handler_type
in (appinfo
.STATIC_FILES
, appinfo
.STATIC_DIR
):
3016 dispatcher
= file_dispatcher
3018 raise InvalidAppConfigError('Unknown handler type "%s"' % handler_type
)
3021 path
= url_map
.GetHandler()
3022 if handler_type
== appinfo
.STATIC_DIR
:
3023 if regex
[-1] == r
'/':
3025 if path
[-1] == os
.path
.sep
:
3027 regex
= '/'.join((re
.escape(regex
), '(.*)'))
3028 if os
.path
.sep
== '\\':
3032 path
= (normpath(path
).replace('\\', '\\\\') +
3033 os
.path
.sep
+ backref
)
3035 url_matcher
.AddURL(regex
,
3038 requires_login
, admin_only
)
3043 class AppConfigCache(object):
3044 """Cache used by LoadAppConfig.
3046 If given to LoadAppConfig instances of this class are used to cache contents
3047 of the app config (app.yaml or app.yml) and the Matcher created from it.
3049 Code outside LoadAppConfig should treat instances of this class as opaque
3050 objects and not access its members.
3059 def LoadAppConfig(root_path
,
3062 static_caching
=True,
3063 read_app_config
=ReadAppConfig
,
3064 create_matcher
=CreateURLMatcherFromMaps
):
3065 """Creates a Matcher instance for an application configuration file.
3067 Raises an InvalidAppConfigError exception if there is anything wrong with
3068 the application configuration file.
3071 root_path: Path to the root of the application to load.
3072 module_dict: Dictionary in which application-loaded modules should be
3073 preserved between requests. This dictionary must be separate from the
3074 sys.modules dictionary.
3075 cache: Instance of AppConfigCache or None.
3076 static_caching: True if browser caching of static files should be allowed.
3077 read_app_config, create_matcher: Used for dependency injection.
3080 tuple: (AppInfoExternal, URLMatcher)
3082 for appinfo_path
in [os
.path
.join(root_path
, 'app.yaml'),
3083 os
.path
.join(root_path
, 'app.yml')]:
3085 if os
.path
.isfile(appinfo_path
):
3086 if cache
is not None:
3087 mtime
= os
.path
.getmtime(appinfo_path
)
3088 if cache
.path
== appinfo_path
and cache
.mtime
== mtime
:
3089 return (cache
.config
, cache
.matcher
)
3091 cache
.config
= cache
.matcher
= cache
.path
= None
3095 config
= read_app_config(appinfo_path
, appinfo
.LoadSingleAppInfo
)
3098 if config
.default_expiration
:
3099 default_expiration
= config
.default_expiration
3101 default_expiration
= '0'
3103 default_expiration
= None
3105 matcher
= create_matcher(root_path
,
3110 FakeFile
.SetSkippedFiles(config
.skip_files
)
3112 if cache
is not None:
3113 cache
.path
= appinfo_path
3114 cache
.config
= config
3115 cache
.matcher
= matcher
3117 return (config
, matcher
)
3118 except gexcept
.AbstractMethod
:
3121 raise AppConfigNotFoundError
3124 def ReadCronConfig(croninfo_path
, parse_cron_config
=croninfo
.LoadSingleCron
):
3125 """Reads cron.yaml file and returns a list of CronEntry instances.
3128 croninfo_path: String containing the path to the cron.yaml file.
3129 parse_cron_config: Used for dependency injection.
3132 A CronInfoExternal object.
3135 If the config file is unreadable, empty or invalid, this function will
3136 raise an InvalidAppConfigError or a MalformedCronConfiguration exception.
3139 croninfo_file
= file(croninfo_path
, 'r')
3141 raise InvalidAppConfigError(
3142 'Cron configuration could not be read from "%s"' % croninfo_path
)
3144 return parse_cron_config(croninfo_file
)
3146 croninfo_file
.close()
3149 def SetupStubs(app_id
, **config
):
3150 """Sets up testing stubs of APIs.
3153 app_id: Application ID being served.
3156 login_url: Relative URL which should be used for handling user login/logout.
3157 datastore_path: Path to the file to store Datastore file stub data in.
3158 history_path: Path to the file to store Datastore history in.
3159 clear_datastore: If the datastore and history should be cleared on startup.
3160 smtp_host: SMTP host used for sending test mail.
3161 smtp_port: SMTP port.
3162 smtp_user: SMTP user.
3163 smtp_password: SMTP password.
3164 enable_sendmail: Whether to use sendmail as an alternative to SMTP.
3165 show_mail_body: Whether to log the body of emails.
3166 remove: Used for dependency injection.
3167 trusted: True if this app can access data belonging to other apps. This
3168 behavior is different from the real app server and should be left False
3169 except for advanced uses of dev_appserver.
3171 login_url
= config
['login_url']
3172 datastore_path
= config
['datastore_path']
3173 history_path
= config
['history_path']
3174 clear_datastore
= config
['clear_datastore']
3175 require_indexes
= config
.get('require_indexes', False)
3176 smtp_host
= config
.get('smtp_host', None)
3177 smtp_port
= config
.get('smtp_port', 25)
3178 smtp_user
= config
.get('smtp_user', '')
3179 smtp_password
= config
.get('smtp_password', '')
3180 enable_sendmail
= config
.get('enable_sendmail', False)
3181 show_mail_body
= config
.get('show_mail_body', False)
3182 remove
= config
.get('remove', os
.remove
)
3183 trusted
= config
.get('trusted', False)
3185 os
.environ
['APPLICATION_ID'] = app_id
3188 for path
in (datastore_path
, history_path
):
3189 if os
.path
.lexists(path
):
3190 logging
.info('Attempting to remove file at %s', path
)
3194 logging
.warning('Removing file failed: %s', e
)
3196 apiproxy_stub_map
.apiproxy
= apiproxy_stub_map
.APIProxyStubMap()
3198 datastore
= datastore_file_stub
.DatastoreFileStub(
3199 app_id
, datastore_path
, history_path
, require_indexes
=require_indexes
,
3201 apiproxy_stub_map
.apiproxy
.RegisterStub('datastore_v3', datastore
)
3203 fixed_login_url
= '%s?%s=%%s' % (login_url
,
3204 dev_appserver_login
.CONTINUE_PARAM
)
3205 fixed_logout_url
= '%s&%s' % (fixed_login_url
,
3206 dev_appserver_login
.LOGOUT_PARAM
)
3208 apiproxy_stub_map
.apiproxy
.RegisterStub(
3210 user_service_stub
.UserServiceStub(login_url
=fixed_login_url
,
3211 logout_url
=fixed_logout_url
))
3213 apiproxy_stub_map
.apiproxy
.RegisterStub(
3215 urlfetch_stub
.URLFetchServiceStub())
3217 apiproxy_stub_map
.apiproxy
.RegisterStub(
3219 mail_stub
.MailServiceStub(smtp_host
,
3223 enable_sendmail
=enable_sendmail
,
3224 show_mail_body
=show_mail_body
))
3226 apiproxy_stub_map
.apiproxy
.RegisterStub(
3228 memcache_stub
.MemcacheServiceStub())
3230 apiproxy_stub_map
.apiproxy
.RegisterStub(
3231 'capability_service',
3232 capability_stub
.CapabilityServiceStub())
3236 from google
.appengine
.api
.images
import images_stub
3237 apiproxy_stub_map
.apiproxy
.RegisterStub(
3239 images_stub
.ImagesServiceStub())
3240 except ImportError, e
:
3241 logging
.warning('Could not initialize images API; you are likely missing '
3242 'the Python "PIL" module. ImportError: %s', e
)
3243 from google
.appengine
.api
.images
import images_not_implemented_stub
3244 apiproxy_stub_map
.apiproxy
.RegisterStub('images',
3245 images_not_implemented_stub
.ImagesNotImplementedServiceStub())
3248 def CreateImplicitMatcher(module_dict
,
3251 create_path_adjuster
=PathAdjuster
,
3252 create_local_dispatcher
=LocalCGIDispatcher
,
3253 create_cgi_dispatcher
=CGIDispatcher
):
3254 """Creates a URLMatcher instance that handles internal URLs.
3256 Used to facilitate handling user login/logout, debugging, info about the
3257 currently running app, etc.
3260 module_dict: Dictionary in the form used by sys.modules.
3261 root_path: Path to the root of the application.
3262 login_url: Relative URL which should be used for handling user login/logout.
3263 create_local_dispatcher: Used for dependency injection.
3266 Instance of URLMatcher with appropriate dispatchers.
3268 url_matcher
= URLMatcher()
3269 path_adjuster
= create_path_adjuster(root_path
)
3271 login_dispatcher
= create_local_dispatcher(sys
.modules
, path_adjuster
,
3272 dev_appserver_login
.main
)
3273 url_matcher
.AddURL(login_url
,
3280 admin_dispatcher
= create_cgi_dispatcher(module_dict
, root_path
,
3282 url_matcher
.AddURL('/_ah/admin(?:/.*)?',
3291 def SetupTemplates(template_dir
):
3292 """Reads debugging console template files and initializes the console.
3294 Does nothing if templates have already been initialized.
3297 template_dir: Path to the directory containing the templates files.
3300 OSError or IOError if any of the template files could not be read.
3302 if ApplicationLoggingHandler
.AreTemplatesInitialized():
3306 header
= open(os
.path
.join(template_dir
, HEADER_TEMPLATE
)).read()
3307 script
= open(os
.path
.join(template_dir
, SCRIPT_TEMPLATE
)).read()
3308 middle
= open(os
.path
.join(template_dir
, MIDDLE_TEMPLATE
)).read()
3309 footer
= open(os
.path
.join(template_dir
, FOOTER_TEMPLATE
)).read()
3310 except (OSError, IOError):
3311 logging
.error('Could not read template files from %s', template_dir
)
3314 ApplicationLoggingHandler
.InitializeTemplates(header
, script
, middle
, footer
)
3317 def CreateServer(root_path
,
3322 require_indexes
=False,
3323 allow_skipped_files
=False,
3324 static_caching
=True,
3325 python_path_list
=sys
.path
,
3326 sdk_dir
=os
.path
.dirname(os
.path
.dirname(google
.__file
__))):
3327 """Creates an new HTTPServer for an application.
3329 The sdk_dir argument must be specified for the directory storing all code for
3330 the SDK so as to allow for the sandboxing of module access to work for any
3331 and all SDK code. While typically this is where the 'google' package lives,
3332 it can be in another location because of API version support.
3335 root_path: String containing the path to the root directory of the
3336 application where the app.yaml file is.
3337 login_url: Relative URL which should be used for handling user login/logout.
3338 port: Port to start the application server on.
3339 template_dir: Path to the directory in which the debug console templates
3341 serve_address: Address on which the server should serve.
3342 require_indexes: True if index.yaml is read-only gospel; default False.
3343 static_caching: True if browser caching of static files should be allowed.
3344 python_path_list: Used for dependency injection.
3345 sdk_dir: Directory where the SDK is stored.
3348 Instance of BaseHTTPServer.HTTPServer that's ready to start accepting.
3350 absolute_root_path
= os
.path
.realpath(root_path
)
3352 SetupTemplates(template_dir
)
3353 FakeFile
.SetAllowedPaths(absolute_root_path
,
3356 FakeFile
.SetAllowSkippedFiles(allow_skipped_files
)
3358 handler_class
= CreateRequestHandler(absolute_root_path
,
3363 if absolute_root_path
not in python_path_list
:
3364 python_path_list
.insert(0, absolute_root_path
)
3366 return BaseHTTPServer
.HTTPServer((serve_address
, port
), handler_class
)