App Engine Python SDK version 1.7.7
[gae.git] / python / google / appengine / tools / devappserver2 / devappserver2.py
blob57f6b970b5b3b485c470ebbdb8ba0f5d10fe7c20
1 #!/usr/bin/env python
3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """The main entry point for the new development server."""
20 import argparse
21 import errno
22 import getpass
23 import itertools
24 import logging
25 import os
26 import sys
27 import tempfile
28 import time
30 from google.appengine.datastore import datastore_stub_util
31 from google.appengine.tools import boolean_action
32 from google.appengine.tools.devappserver2.admin import admin_server
33 from google.appengine.tools.devappserver2 import api_server
34 from google.appengine.tools.devappserver2 import application_configuration
35 from google.appengine.tools.devappserver2 import dispatcher
36 from google.appengine.tools.devappserver2 import login
37 from google.appengine.tools.devappserver2 import runtime_config_pb2
38 from google.appengine.tools.devappserver2 import shutdown
39 from google.appengine.tools.devappserver2 import update_checker
40 from google.appengine.tools.devappserver2 import wsgi_request_info
42 # Initialize logging early -- otherwise some library packages may
43 # pre-empt our log formatting. NOTE: the level is provisional; it may
44 # be changed in main() based on the --debug flag.
45 logging.basicConfig(
46 level=logging.INFO,
47 format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
49 # Valid choices for --log_level and their corresponding constants in
50 # runtime_config_pb2.Config.stderr_log_level.
51 _LOG_LEVEL_TO_RUNTIME_CONSTANT = {
52 'debug': 0,
53 'info': 1,
54 'warning': 2,
55 'error': 3,
56 'critical': 4,
59 # Valid choices for --dev_appserver_log_level and their corresponding Python
60 # logging levels
61 _LOG_LEVEL_TO_PYTHON_CONSTANT = {
62 'debug': logging.DEBUG,
63 'info': logging.INFO,
64 'warning': logging.WARNING,
65 'error': logging.ERROR,
66 'critical': logging.CRITICAL,
70 def _generate_storage_paths(app_id):
71 """Yield an infinite sequence of possible storage paths."""
72 try:
73 user_name = getpass.getuser()
74 except Exception: # The possible set of exceptions is not documented.
75 user_format = ''
76 else:
77 user_format = '.%s' % user_name
79 tempdir = tempfile.gettempdir()
80 yield os.path.join(tempdir, 'appengine.%s%s' % (app_id, user_format))
81 for i in itertools.count(1):
82 yield os.path.join(tempdir, 'appengine.%s%s.%d' % (app_id, user_format, i))
85 def _get_storage_path(path, app_id):
86 """Returns a path to the directory where stub data can be stored."""
87 _, _, app_id = app_id.replace(':', '_').rpartition('~')
88 if path is None:
89 for path in _generate_storage_paths(app_id):
90 try:
91 os.mkdir(path, 0700)
92 except OSError, e:
93 if e.errno == errno.EEXIST:
94 # Check that the directory is only accessable by the current user to
95 # protect against an attacker creating the directory in advance in
96 # order to access any created files. Windows has per-user temporary
97 # directories and st_mode does not include per-user permission
98 # information so assume that it is safe.
99 if sys.platform == 'win32' or (
100 (os.stat(path).st_mode & 0777) == 0700 and os.path.isdir(path)):
101 return path
102 else:
103 continue
104 raise
105 else:
106 return path
107 elif not os.path.exists(path):
108 os.mkdir(path)
109 return path
110 elif not os.path.isdir(path):
111 raise IOError('the given storage path %r is a file, a directory was '
112 'expected' % path)
113 else:
114 return path
117 class PortParser(object):
118 """A parser for ints that represent ports."""
120 def __init__(self, allow_port_zero=True):
121 self._min_port = 0 if allow_port_zero else 1
123 def __call__(self, value):
124 try:
125 port = int(value)
126 except ValueError:
127 raise argparse.ArgumentTypeError('Invalid port: %r' % value)
128 if port < self._min_port or port >= (1 << 16):
129 raise argparse.ArgumentTypeError('Invalid port: %d' % port)
130 return port
133 def parse_max_server_instances(value):
134 """Returns the parsed value for the --max_server_instances flag.
136 Args:
137 value: A str containing the flag value for parse. The format should follow
138 one of the following examples:
139 1. "5" - All servers are limited to 5 instances.
140 2. "default:3,backend:20" - The default server can have 3 instances,
141 "backend" can have 20 instances and all other servers are
142 unaffected.
144 if ':' not in value:
145 try:
146 max_server_instances = int(value)
147 except ValueError:
148 raise argparse.ArgumentTypeError('Invalid instance count: %r' % value)
149 else:
150 if not max_server_instances:
151 raise argparse.ArgumentTypeError(
152 'Cannot specify zero instances for all servers')
153 return max_server_instances
154 else:
155 server_to_max_instances = {}
156 for server_instance_max in value.split(','):
157 try:
158 server_name, max_instances = server_instance_max.split(':')
159 max_instances = int(max_instances)
160 except ValueError:
161 raise argparse.ArgumentTypeError(
162 'Expected "server:max_instances": %r' % server_instance_max)
163 else:
164 server_name = server_name.strip()
165 if server_name in server_to_max_instances:
166 raise argparse.ArgumentTypeError(
167 'Duplicate max instance value: %r' % server_name)
168 server_to_max_instances[server_name] = max_instances
169 return server_to_max_instances
172 def create_command_line_parser():
173 """Returns an argparse.ArgumentParser to parse command line arguments."""
174 # TODO: Add more robust argument validation. Consider what flags
175 # are actually needed.
177 parser = argparse.ArgumentParser(
178 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
179 parser.add_argument('yaml_files', nargs='+')
181 common_group = parser.add_argument_group('Common')
182 common_group.add_argument(
183 '--host', default='localhost',
184 help='host name to which application servers should bind')
185 common_group.add_argument(
186 '--port', type=PortParser(), default=8080,
187 help='lowest port to which application servers should bind')
188 common_group.add_argument(
189 '--admin_host', default='localhost',
190 help='host name to which the admin server should bind')
191 common_group.add_argument(
192 '--admin_port', type=PortParser(), default=8000,
193 help='port to which the admin server should bind')
194 common_group.add_argument(
195 '--storage_path', metavar='PATH',
196 type=os.path.expanduser,
197 help='path to the data (datastore, blobstore, etc.) associated with the '
198 'application.')
199 common_group.add_argument(
200 '--log_level', default='info',
201 choices=_LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(),
202 help='the log level below which logging messages generated by '
203 'application code will not be displayed on the console')
204 common_group.add_argument(
205 '--max_server_instances',
206 type=parse_max_server_instances,
207 help='the maximum number of runtime instances that can be started for a '
208 'particular server - the value can be an integer, in what case all '
209 'servers are limited to that number of instances or a comma-seperated '
210 'list of server:max_instances e.g. "default:5,backend:3"')
211 common_group.add_argument(
212 '--use_mtime_file_watcher',
213 action=boolean_action.BooleanAction,
214 const=True,
215 default=False,
216 help='use mtime polling for detecting source code changes - useful if '
217 'modifying code from a remote machine using a distributed file system')
221 # Python
222 python_group = parser.add_argument_group('Python')
223 python_group.add_argument(
224 '--python_startup_script',
225 help='the script to run at the startup of new Python runtime instances '
226 '(useful for tools such as debuggers.')
227 python_group.add_argument(
228 '--python_startup_args',
229 help='the arguments made available to the script specified in '
230 '--python_startup_script.')
232 # Blobstore
233 blobstore_group = parser.add_argument_group('Blobstore API')
234 blobstore_group.add_argument(
235 '--blobstore_path',
236 type=os.path.expanduser,
237 help='path to directory used to store blob contents '
238 '(defaults to a subdirectory of --storage_path if not set)',
239 default=None)
241 # Cloud SQL
242 cloud_sql_group = parser.add_argument_group('Cloud SQL')
243 cloud_sql_group.add_argument(
244 '--mysql_host',
245 default='localhost',
246 help='host name of a running MySQL server used for simulated Google '
247 'Cloud SQL storage')
248 cloud_sql_group.add_argument(
249 '--mysql_port', type=PortParser(allow_port_zero=False),
250 default=3306,
251 help='port number of a running MySQL server used for simulated Google '
252 'Cloud SQL storage')
253 cloud_sql_group.add_argument(
254 '--mysql_user',
255 default='',
256 help='username to use when connecting to the MySQL server specified in '
257 '--mysql_host and --mysql_port or --mysql_socket')
258 cloud_sql_group.add_argument(
259 '--mysql_password',
260 default='',
261 help='passpord to use when connecting to the MySQL server specified in '
262 '--mysql_host and --mysql_port or --mysql_socket')
263 cloud_sql_group.add_argument(
264 '--mysql_socket',
265 help='path to a Unix socket file to use when connecting to a running '
266 'MySQL server used for simulated Google Cloud SQL storage')
268 # Datastore
269 datastore_group = parser.add_argument_group('Datastore API')
270 datastore_group.add_argument(
271 '--datastore_path',
272 type=os.path.expanduser,
273 default=None,
274 help='path to a file used to store datastore contents '
275 '(defaults to a file in --storage_path if not set)',)
276 datastore_group.add_argument('--clear_datastore',
277 action=boolean_action.BooleanAction,
278 const=True,
279 default=False,
280 help='clear the datastore on startup')
281 datastore_group.add_argument(
282 '--datastore_consistency_policy',
283 default='time',
284 choices=['consistent', 'random', 'time'],
285 help='the policy to apply when deciding whether a datastore write should '
286 'appear in global queries')
287 datastore_group.add_argument(
288 '--require_indexes',
289 action=boolean_action.BooleanAction,
290 const=True,
291 default=False,
292 help='generate an error on datastore queries that '
293 'requires a composite index not found in index.yaml')
294 datastore_group.add_argument(
295 '--auto_id_policy',
296 default=datastore_stub_util.SCATTERED,
297 choices=[datastore_stub_util.SEQUENTIAL,
298 datastore_stub_util.SCATTERED],
299 help='the type of sequence from which the datastore stub '
300 'assigns automatic IDs. NOTE: Sequential IDs are '
301 'deprecated. This flag will be removed in a future '
302 'release. Please do not rely on sequential IDs in your '
303 'tests.')
305 # Logs
306 logs_group = parser.add_argument_group('Logs API')
307 logs_group.add_argument(
308 '--logs_path', default=None,
309 help='path to a file used to store request logs (defaults to a file in '
310 '--storage_path if not set)',)
312 # Mail
313 mail_group = parser.add_argument_group('Mail API')
314 mail_group.add_argument(
315 '--show_mail_body',
316 action=boolean_action.BooleanAction,
317 const=True,
318 default=False,
319 help='logs the contents of e-mails sent using the Mail API')
320 mail_group.add_argument(
321 '--enable_sendmail',
322 action=boolean_action.BooleanAction,
323 const=True,
324 default=False,
325 help='use the "sendmail" tool to transmit e-mail sent '
326 'using the Mail API (ignored if --smpt_host is set)')
327 mail_group.add_argument(
328 '--smtp_host', default='',
329 help='host name of an SMTP server to use to transmit '
330 'e-mail sent using the Mail API')
331 mail_group.add_argument(
332 '--smtp_port', default=25,
333 type=PortParser(allow_port_zero=False),
334 help='port number of an SMTP server to use to transmit '
335 'e-mail sent using the Mail API (ignored if --smtp_host '
336 'is not set)')
337 mail_group.add_argument(
338 '--smtp_user', default='',
339 help='username to use when connecting to the SMTP server '
340 'specified in --smtp_host and --smtp_port')
341 mail_group.add_argument(
342 '--smtp_password', default='',
343 help='password to use when connecting to the SMTP server '
344 'specified in --smtp_host and --smtp_port')
346 # Matcher
347 prospective_search_group = parser.add_argument_group('Prospective Search API')
348 prospective_search_group.add_argument(
349 '--prospective_search_path', default=None,
350 type=os.path.expanduser,
351 help='path to a file used to store the prospective '
352 'search subscription index (defaults to a file in '
353 '--storage_path if not set)')
354 prospective_search_group.add_argument(
355 '--clear_prospective_search',
356 action=boolean_action.BooleanAction,
357 const=True,
358 default=False,
359 help='clear the prospective search subscription index')
361 # Search
362 search_group = parser.add_argument_group('Search API')
363 search_group.add_argument(
364 '--search_indexes_path', default=None,
365 type=os.path.expanduser,
366 help='path to a file used to store search indexes '
367 '(defaults to a file in --storage_path if not set)',)
368 search_group.add_argument(
369 '--clear_search_indexes',
370 action=boolean_action.BooleanAction,
371 const=True,
372 default=False,
373 help='clear the search indexes')
375 # Taskqueue
376 taskqueue_group = parser.add_argument_group('Task Queue API')
377 taskqueue_group.add_argument(
378 '--enable_task_running',
379 action=boolean_action.BooleanAction,
380 const=True,
381 default=True,
382 help='run "push" tasks created using the taskqueue API automatically')
384 # Misc
385 misc_group = parser.add_argument_group('Miscellaneous')
386 misc_group.add_argument(
387 '--api_port', type=PortParser(), default=0,
388 help='port to which the server for API calls should bind')
389 misc_group.add_argument(
390 '--automatic_restart',
391 action=boolean_action.BooleanAction,
392 const=True,
393 default=True,
394 help=('restart instances automatically when files relevant to their '
395 'server are changed'))
396 misc_group.add_argument(
397 '--dev_appserver_log_level', default='info',
398 choices=_LOG_LEVEL_TO_PYTHON_CONSTANT.keys(),
399 help='the log level below which logging messages generated by '
400 'the development server will not be displayed on the console (this '
401 'flag is more useful for diagnosing problems in dev_appserver.py rather '
402 'than in application code)')
403 misc_group.add_argument(
404 '--skip_sdk_update_check',
405 action=boolean_action.BooleanAction,
406 const=True,
407 default=False,
408 help='skip checking for SDK updates (if false, use .appcfg_nag to '
409 'decide)')
412 return parser
414 PARSER = create_command_line_parser()
417 def _clear_datastore_storage(datastore_path):
418 """Delete the datastore storage file at the given path."""
419 # lexists() returns True for broken symlinks, where exists() returns False.
420 if os.path.lexists(datastore_path):
421 try:
422 os.remove(datastore_path)
423 except OSError, e:
424 logging.warning('Failed to remove datastore file %r: %s',
425 datastore_path,
429 def _clear_prospective_search_storage(prospective_search_path):
430 """Delete the perspective search storage file at the given path."""
431 # lexists() returns True for broken symlinks, where exists() returns False.
432 if os.path.lexists(prospective_search_path):
433 try:
434 os.remove(prospective_search_path)
435 except OSError, e:
436 logging.warning('Failed to remove prospective search file %r: %s',
437 prospective_search_path,
441 def _clear_search_indexes_storage(search_index_path):
442 """Delete the search indexes storage file at the given path."""
443 # lexists() returns True for broken symlinks, where exists() returns False.
444 if os.path.lexists(search_index_path):
445 try:
446 os.remove(search_index_path)
447 except OSError, e:
448 logging.warning('Failed to remove search indexes file %r: %s',
449 search_index_path,
453 def _setup_environ(app_id):
454 """Sets up the os.environ dictionary for the front-end server and API server.
456 This function should only be called once.
458 Args:
459 app_id: The id of the application.
461 os.environ['APPLICATION_ID'] = app_id
464 class DevelopmentServer(object):
465 """Encapsulates the logic for the development server.
467 Only a single instance of the class may be created per process. See
468 _setup_environ.
471 def __init__(self):
472 # A list of servers that are currently running.
473 self._running_servers = []
474 self._server_to_port = {}
476 def server_to_address(self, server_name, instance=None):
477 """Returns the address of a server."""
478 if server_name is None:
479 return self._dispatcher.dispatch_address
480 return self._dispatcher.get_hostname(
481 server_name,
482 self._dispatcher.get_default_version(server_name),
483 instance)
485 def start(self, options):
486 """Start devappserver2 servers based on the provided command line arguments.
488 Args:
489 options: An argparse.Namespace containing the command line arguments.
491 logging.getLogger().setLevel(
492 _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
494 configuration = application_configuration.ApplicationConfiguration(
495 options.yaml_files)
497 if options.skip_sdk_update_check:
498 logging.info('Skipping SDK update check.')
499 else:
500 update_checker.check_for_updates(configuration)
502 if options.port == 0:
503 logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
504 '--port=0')
506 _setup_environ(configuration.app_id)
508 python_config = runtime_config_pb2.PythonConfig()
509 if options.python_startup_script:
510 python_config.startup_script = os.path.abspath(
511 options.python_startup_script)
512 if options.python_startup_args:
513 python_config.startup_args = options.python_startup_args
515 cloud_sql_config = runtime_config_pb2.CloudSQL()
516 cloud_sql_config.mysql_host = options.mysql_host
517 cloud_sql_config.mysql_port = options.mysql_port
518 cloud_sql_config.mysql_user = options.mysql_user
519 cloud_sql_config.mysql_password = options.mysql_password
520 if options.mysql_socket:
521 cloud_sql_config.mysql_socket = options.mysql_socket
523 if options.max_server_instances is None:
524 server_to_max_instances = {}
525 elif isinstance(options.max_server_instances, int):
526 server_to_max_instances = {
527 server_configuration.server_name: options.max_server_instances
528 for server_configuration in configuration.servers}
529 else:
530 server_to_max_instances = options.max_server_instances
532 self._dispatcher = dispatcher.Dispatcher(
533 configuration,
534 options.host,
535 options.port,
536 _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
538 python_config,
539 cloud_sql_config,
540 server_to_max_instances,
541 options.use_mtime_file_watcher,
542 options.automatic_restart)
544 request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
546 storage_path = _get_storage_path(options.storage_path, configuration.app_id)
547 datastore_path = options.datastore_path or os.path.join(storage_path,
548 'datastore.db')
549 logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
550 xsrf_path = os.path.join(storage_path, 'xsrf')
552 search_index_path = options.search_indexes_path or os.path.join(
553 storage_path, 'search_indexes')
555 prospective_search_path = options.prospective_search_path or os.path.join(
556 storage_path, 'prospective-search')
558 blobstore_path = options.blobstore_path or os.path.join(storage_path,
559 'blobs')
561 if options.clear_datastore:
562 _clear_datastore_storage(datastore_path)
564 if options.clear_prospective_search:
565 _clear_prospective_search_storage(prospective_search_path)
567 if options.clear_search_indexes:
568 _clear_search_indexes_storage(search_index_path)
570 if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
571 logging.warn("--auto_id_policy='sequential' is deprecated. This option "
572 "will be removed in a future release.")
574 application_address = '%s' % options.host
575 if options.port and options.port != 80:
576 application_address += ':' + str(options.port)
578 user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
579 login.CONTINUE_PARAM)
580 user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
581 login.LOGOUT_ACTION)
583 if options.datastore_consistency_policy == 'time':
584 consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
585 elif options.datastore_consistency_policy == 'random':
586 consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
587 elif options.datastore_consistency_policy == 'consistent':
588 consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
589 else:
590 assert 0, ('unknown consistency policy: %r' %
591 options.datastore_consistency_policy)
593 api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
594 configuration.app_id, datastore_path)
595 api_server.setup_stubs(
596 request_data=request_data,
597 app_id=configuration.app_id,
598 application_root=configuration.servers[0].application_root,
599 # The "trusted" flag is only relevant for Google administrative
600 # applications.
601 trusted=getattr(options, 'trusted', False),
602 blobstore_path=blobstore_path,
603 datastore_path=datastore_path,
604 datastore_consistency=consistency,
605 datastore_require_indexes=options.require_indexes,
606 datastore_auto_id_policy=options.auto_id_policy,
607 images_host_prefix='http://%s' % application_address,
608 logs_path=logs_path,
609 mail_smtp_host=options.smtp_host,
610 mail_smtp_port=options.smtp_port,
611 mail_smtp_user=options.smtp_user,
612 mail_smtp_password=options.smtp_password,
613 mail_enable_sendmail=options.enable_sendmail,
614 mail_show_mail_body=options.show_mail_body,
615 matcher_prospective_search_path=prospective_search_path,
616 search_index_path=search_index_path,
617 taskqueue_auto_run_tasks=options.enable_task_running,
618 taskqueue_default_http_server=application_address,
619 user_login_url=user_login_url,
620 user_logout_url=user_logout_url)
622 # The APIServer must bind to localhost because that is what the runtime
623 # instances talk to.
624 apis = api_server.APIServer('localhost', options.api_port,
625 configuration.app_id)
626 apis.start()
627 self._running_servers.append(apis)
629 self._running_servers.append(self._dispatcher)
630 self._dispatcher.start(apis.port, request_data)
632 admin = admin_server.AdminServer(options.admin_host, options.admin_port,
633 self._dispatcher, configuration, xsrf_path)
634 admin.start()
635 self._running_servers.append(admin)
637 def stop(self):
638 """Stops all running devappserver2 servers."""
639 while self._running_servers:
640 self._running_servers.pop().quit()
643 def main():
644 shutdown.install_signal_handlers()
645 # The timezone must be set in the devappserver2 process rather than just in
646 # the runtime so printed log timestamps are consistent and the taskqueue stub
647 # expects the timezone to be UTC. The runtime inherits the environment.
648 os.environ['TZ'] = 'UTC'
649 if hasattr(time, 'tzset'):
650 # time.tzet() should be called on Unix, but doesn't exist on Windows.
651 time.tzset()
652 options = PARSER.parse_args()
653 dev_server = DevelopmentServer()
654 try:
655 dev_server.start(options)
656 shutdown.wait_until_shutdown()
657 finally:
658 dev_server.stop()
661 if __name__ == '__main__':
662 main()