3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """The main entry point for the new development server."""
30 from google
.appengine
.datastore
import datastore_stub_util
31 from google
.appengine
.tools
import boolean_action
32 from google
.appengine
.tools
.devappserver2
.admin
import admin_server
33 from google
.appengine
.tools
.devappserver2
import api_server
34 from google
.appengine
.tools
.devappserver2
import application_configuration
35 from google
.appengine
.tools
.devappserver2
import dispatcher
36 from google
.appengine
.tools
.devappserver2
import login
37 from google
.appengine
.tools
.devappserver2
import runtime_config_pb2
38 from google
.appengine
.tools
.devappserver2
import shutdown
39 from google
.appengine
.tools
.devappserver2
import update_checker
40 from google
.appengine
.tools
.devappserver2
import wsgi_request_info
42 # Initialize logging early -- otherwise some library packages may
43 # pre-empt our log formatting. NOTE: the level is provisional; it may
44 # be changed in main() based on the --debug flag.
47 format
='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
49 # Valid choices for --log_level and their corresponding constants in
50 # runtime_config_pb2.Config.stderr_log_level.
51 _LOG_LEVEL_TO_RUNTIME_CONSTANT
= {
59 # Valid choices for --dev_appserver_log_level and their corresponding Python
61 _LOG_LEVEL_TO_PYTHON_CONSTANT
= {
62 'debug': logging
.DEBUG
,
64 'warning': logging
.WARNING
,
65 'error': logging
.ERROR
,
66 'critical': logging
.CRITICAL
,
70 def _generate_storage_paths(app_id
):
71 """Yield an infinite sequence of possible storage paths."""
73 user_name
= getpass
.getuser()
74 except Exception: # The possible set of exceptions is not documented.
77 user_format
= '.%s' % user_name
79 tempdir
= tempfile
.gettempdir()
80 yield os
.path
.join(tempdir
, 'appengine.%s%s' % (app_id
, user_format
))
81 for i
in itertools
.count(1):
82 yield os
.path
.join(tempdir
, 'appengine.%s%s.%d' % (app_id
, user_format
, i
))
85 def _get_storage_path(path
, app_id
):
86 """Returns a path to the directory where stub data can be stored."""
87 _
, _
, app_id
= app_id
.replace(':', '_').rpartition('~')
89 for path
in _generate_storage_paths(app_id
):
93 if e
.errno
== errno
.EEXIST
:
94 # Check that the directory is only accessable by the current user to
95 # protect against an attacker creating the directory in advance in
96 # order to access any created files. Windows has per-user temporary
97 # directories and st_mode does not include per-user permission
98 # information so assume that it is safe.
99 if sys
.platform
== 'win32' or (
100 (os
.stat(path
).st_mode
& 0777) == 0700 and os
.path
.isdir(path
)):
107 elif not os
.path
.exists(path
):
110 elif not os
.path
.isdir(path
):
111 raise IOError('the given storage path %r is a file, a directory was '
117 class PortParser(object):
118 """A parser for ints that represent ports."""
120 def __init__(self
, allow_port_zero
=True):
121 self
._min
_port
= 0 if allow_port_zero
else 1
123 def __call__(self
, value
):
127 raise argparse
.ArgumentTypeError('Invalid port: %r' % value
)
128 if port
< self
._min
_port
or port
>= (1 << 16):
129 raise argparse
.ArgumentTypeError('Invalid port: %d' % port
)
133 def parse_max_server_instances(value
):
134 """Returns the parsed value for the --max_server_instances flag.
137 value: A str containing the flag value for parse. The format should follow
138 one of the following examples:
139 1. "5" - All servers are limited to 5 instances.
140 2. "default:3,backend:20" - The default server can have 3 instances,
141 "backend" can have 20 instances and all other servers are
146 max_server_instances
= int(value
)
148 raise argparse
.ArgumentTypeError('Invalid instance count: %r' % value
)
150 if not max_server_instances
:
151 raise argparse
.ArgumentTypeError(
152 'Cannot specify zero instances for all servers')
153 return max_server_instances
155 server_to_max_instances
= {}
156 for server_instance_max
in value
.split(','):
158 server_name
, max_instances
= server_instance_max
.split(':')
159 max_instances
= int(max_instances
)
161 raise argparse
.ArgumentTypeError(
162 'Expected "server:max_instances": %r' % server_instance_max
)
164 server_name
= server_name
.strip()
165 if server_name
in server_to_max_instances
:
166 raise argparse
.ArgumentTypeError(
167 'Duplicate max instance value: %r' % server_name
)
168 server_to_max_instances
[server_name
] = max_instances
169 return server_to_max_instances
172 def create_command_line_parser():
173 """Returns an argparse.ArgumentParser to parse command line arguments."""
174 # TODO: Add more robust argument validation. Consider what flags
175 # are actually needed.
177 parser
= argparse
.ArgumentParser(
178 formatter_class
=argparse
.ArgumentDefaultsHelpFormatter
)
179 parser
.add_argument('yaml_files', nargs
='+')
181 common_group
= parser
.add_argument_group('Common')
182 common_group
.add_argument(
183 '--host', default
='localhost',
184 help='host name to which application servers should bind')
185 common_group
.add_argument(
186 '--port', type=PortParser(), default
=8080,
187 help='lowest port to which application servers should bind')
188 common_group
.add_argument(
189 '--admin_host', default
='localhost',
190 help='host name to which the admin server should bind')
191 common_group
.add_argument(
192 '--admin_port', type=PortParser(), default
=8000,
193 help='port to which the admin server should bind')
194 common_group
.add_argument(
195 '--storage_path', metavar
='PATH',
196 type=os
.path
.expanduser
,
197 help='path to the data (datastore, blobstore, etc.) associated with the '
199 common_group
.add_argument(
200 '--log_level', default
='info',
201 choices
=_LOG_LEVEL_TO_RUNTIME_CONSTANT
.keys(),
202 help='the log level below which logging messages generated by '
203 'application code will not be displayed on the console')
204 common_group
.add_argument(
205 '--max_server_instances',
206 type=parse_max_server_instances
,
207 help='the maximum number of runtime instances that can be started for a '
208 'particular server - the value can be an integer, in what case all '
209 'servers are limited to that number of instances or a comma-seperated '
210 'list of server:max_instances e.g. "default:5,backend:3"')
211 common_group
.add_argument(
212 '--use_mtime_file_watcher',
213 action
=boolean_action
.BooleanAction
,
216 help='use mtime polling for detecting source code changes - useful if '
217 'modifying code from a remote machine using a distributed file system')
222 python_group
= parser
.add_argument_group('Python')
223 python_group
.add_argument(
224 '--python_startup_script',
225 help='the script to run at the startup of new Python runtime instances '
226 '(useful for tools such as debuggers.')
227 python_group
.add_argument(
228 '--python_startup_args',
229 help='the arguments made available to the script specified in '
230 '--python_startup_script.')
233 blobstore_group
= parser
.add_argument_group('Blobstore API')
234 blobstore_group
.add_argument(
236 type=os
.path
.expanduser
,
237 help='path to directory used to store blob contents '
238 '(defaults to a subdirectory of --storage_path if not set)',
242 cloud_sql_group
= parser
.add_argument_group('Cloud SQL')
243 cloud_sql_group
.add_argument(
246 help='host name of a running MySQL server used for simulated Google '
248 cloud_sql_group
.add_argument(
249 '--mysql_port', type=PortParser(allow_port_zero
=False),
251 help='port number of a running MySQL server used for simulated Google '
253 cloud_sql_group
.add_argument(
256 help='username to use when connecting to the MySQL server specified in '
257 '--mysql_host and --mysql_port or --mysql_socket')
258 cloud_sql_group
.add_argument(
261 help='passpord to use when connecting to the MySQL server specified in '
262 '--mysql_host and --mysql_port or --mysql_socket')
263 cloud_sql_group
.add_argument(
265 help='path to a Unix socket file to use when connecting to a running '
266 'MySQL server used for simulated Google Cloud SQL storage')
269 datastore_group
= parser
.add_argument_group('Datastore API')
270 datastore_group
.add_argument(
272 type=os
.path
.expanduser
,
274 help='path to a file used to store datastore contents '
275 '(defaults to a file in --storage_path if not set)',)
276 datastore_group
.add_argument('--clear_datastore',
277 action
=boolean_action
.BooleanAction
,
280 help='clear the datastore on startup')
281 datastore_group
.add_argument(
282 '--datastore_consistency_policy',
284 choices
=['consistent', 'random', 'time'],
285 help='the policy to apply when deciding whether a datastore write should '
286 'appear in global queries')
287 datastore_group
.add_argument(
289 action
=boolean_action
.BooleanAction
,
292 help='generate an error on datastore queries that '
293 'requires a composite index not found in index.yaml')
294 datastore_group
.add_argument(
296 default
=datastore_stub_util
.SCATTERED
,
297 choices
=[datastore_stub_util
.SEQUENTIAL
,
298 datastore_stub_util
.SCATTERED
],
299 help='the type of sequence from which the datastore stub '
300 'assigns automatic IDs. NOTE: Sequential IDs are '
301 'deprecated. This flag will be removed in a future '
302 'release. Please do not rely on sequential IDs in your '
306 logs_group
= parser
.add_argument_group('Logs API')
307 logs_group
.add_argument(
308 '--logs_path', default
=None,
309 help='path to a file used to store request logs (defaults to a file in '
310 '--storage_path if not set)',)
313 mail_group
= parser
.add_argument_group('Mail API')
314 mail_group
.add_argument(
316 action
=boolean_action
.BooleanAction
,
319 help='logs the contents of e-mails sent using the Mail API')
320 mail_group
.add_argument(
322 action
=boolean_action
.BooleanAction
,
325 help='use the "sendmail" tool to transmit e-mail sent '
326 'using the Mail API (ignored if --smpt_host is set)')
327 mail_group
.add_argument(
328 '--smtp_host', default
='',
329 help='host name of an SMTP server to use to transmit '
330 'e-mail sent using the Mail API')
331 mail_group
.add_argument(
332 '--smtp_port', default
=25,
333 type=PortParser(allow_port_zero
=False),
334 help='port number of an SMTP server to use to transmit '
335 'e-mail sent using the Mail API (ignored if --smtp_host '
337 mail_group
.add_argument(
338 '--smtp_user', default
='',
339 help='username to use when connecting to the SMTP server '
340 'specified in --smtp_host and --smtp_port')
341 mail_group
.add_argument(
342 '--smtp_password', default
='',
343 help='password to use when connecting to the SMTP server '
344 'specified in --smtp_host and --smtp_port')
347 prospective_search_group
= parser
.add_argument_group('Prospective Search API')
348 prospective_search_group
.add_argument(
349 '--prospective_search_path', default
=None,
350 type=os
.path
.expanduser
,
351 help='path to a file used to store the prospective '
352 'search subscription index (defaults to a file in '
353 '--storage_path if not set)')
354 prospective_search_group
.add_argument(
355 '--clear_prospective_search',
356 action
=boolean_action
.BooleanAction
,
359 help='clear the prospective search subscription index')
362 search_group
= parser
.add_argument_group('Search API')
363 search_group
.add_argument(
364 '--search_indexes_path', default
=None,
365 type=os
.path
.expanduser
,
366 help='path to a file used to store search indexes '
367 '(defaults to a file in --storage_path if not set)',)
368 search_group
.add_argument(
369 '--clear_search_indexes',
370 action
=boolean_action
.BooleanAction
,
373 help='clear the search indexes')
376 taskqueue_group
= parser
.add_argument_group('Task Queue API')
377 taskqueue_group
.add_argument(
378 '--enable_task_running',
379 action
=boolean_action
.BooleanAction
,
382 help='run "push" tasks created using the taskqueue API automatically')
385 misc_group
= parser
.add_argument_group('Miscellaneous')
386 misc_group
.add_argument(
387 '--api_port', type=PortParser(), default
=0,
388 help='port to which the server for API calls should bind')
389 misc_group
.add_argument(
390 '--automatic_restart',
391 action
=boolean_action
.BooleanAction
,
394 help=('restart instances automatically when files relevant to their '
395 'server are changed'))
396 misc_group
.add_argument(
397 '--dev_appserver_log_level', default
='info',
398 choices
=_LOG_LEVEL_TO_PYTHON_CONSTANT
.keys(),
399 help='the log level below which logging messages generated by '
400 'the development server will not be displayed on the console (this '
401 'flag is more useful for diagnosing problems in dev_appserver.py rather '
402 'than in application code)')
403 misc_group
.add_argument(
404 '--skip_sdk_update_check',
405 action
=boolean_action
.BooleanAction
,
408 help='skip checking for SDK updates (if false, use .appcfg_nag to '
414 PARSER
= create_command_line_parser()
417 def _clear_datastore_storage(datastore_path
):
418 """Delete the datastore storage file at the given path."""
419 # lexists() returns True for broken symlinks, where exists() returns False.
420 if os
.path
.lexists(datastore_path
):
422 os
.remove(datastore_path
)
424 logging
.warning('Failed to remove datastore file %r: %s',
429 def _clear_prospective_search_storage(prospective_search_path
):
430 """Delete the perspective search storage file at the given path."""
431 # lexists() returns True for broken symlinks, where exists() returns False.
432 if os
.path
.lexists(prospective_search_path
):
434 os
.remove(prospective_search_path
)
436 logging
.warning('Failed to remove prospective search file %r: %s',
437 prospective_search_path
,
441 def _clear_search_indexes_storage(search_index_path
):
442 """Delete the search indexes storage file at the given path."""
443 # lexists() returns True for broken symlinks, where exists() returns False.
444 if os
.path
.lexists(search_index_path
):
446 os
.remove(search_index_path
)
448 logging
.warning('Failed to remove search indexes file %r: %s',
453 def _setup_environ(app_id
):
454 """Sets up the os.environ dictionary for the front-end server and API server.
456 This function should only be called once.
459 app_id: The id of the application.
461 os
.environ
['APPLICATION_ID'] = app_id
464 class DevelopmentServer(object):
465 """Encapsulates the logic for the development server.
467 Only a single instance of the class may be created per process. See
472 # A list of servers that are currently running.
473 self
._running
_servers
= []
474 self
._server
_to
_port
= {}
476 def server_to_address(self
, server_name
, instance
=None):
477 """Returns the address of a server."""
478 if server_name
is None:
479 return self
._dispatcher
.dispatch_address
480 return self
._dispatcher
.get_hostname(
482 self
._dispatcher
.get_default_version(server_name
),
485 def start(self
, options
):
486 """Start devappserver2 servers based on the provided command line arguments.
489 options: An argparse.Namespace containing the command line arguments.
491 logging
.getLogger().setLevel(
492 _LOG_LEVEL_TO_PYTHON_CONSTANT
[options
.dev_appserver_log_level
])
494 configuration
= application_configuration
.ApplicationConfiguration(
497 if options
.skip_sdk_update_check
:
498 logging
.info('Skipping SDK update check.')
500 update_checker
.check_for_updates(configuration
)
502 if options
.port
== 0:
503 logging
.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
506 _setup_environ(configuration
.app_id
)
508 python_config
= runtime_config_pb2
.PythonConfig()
509 if options
.python_startup_script
:
510 python_config
.startup_script
= os
.path
.abspath(
511 options
.python_startup_script
)
512 if options
.python_startup_args
:
513 python_config
.startup_args
= options
.python_startup_args
515 cloud_sql_config
= runtime_config_pb2
.CloudSQL()
516 cloud_sql_config
.mysql_host
= options
.mysql_host
517 cloud_sql_config
.mysql_port
= options
.mysql_port
518 cloud_sql_config
.mysql_user
= options
.mysql_user
519 cloud_sql_config
.mysql_password
= options
.mysql_password
520 if options
.mysql_socket
:
521 cloud_sql_config
.mysql_socket
= options
.mysql_socket
523 if options
.max_server_instances
is None:
524 server_to_max_instances
= {}
525 elif isinstance(options
.max_server_instances
, int):
526 server_to_max_instances
= {
527 server_configuration
.server_name
: options
.max_server_instances
528 for server_configuration
in configuration
.servers
}
530 server_to_max_instances
= options
.max_server_instances
532 self
._dispatcher
= dispatcher
.Dispatcher(
536 _LOG_LEVEL_TO_RUNTIME_CONSTANT
[options
.log_level
],
540 server_to_max_instances
,
541 options
.use_mtime_file_watcher
,
542 options
.automatic_restart
)
544 request_data
= wsgi_request_info
.WSGIRequestInfo(self
._dispatcher
)
546 storage_path
= _get_storage_path(options
.storage_path
, configuration
.app_id
)
547 datastore_path
= options
.datastore_path
or os
.path
.join(storage_path
,
549 logs_path
= options
.logs_path
or os
.path
.join(storage_path
, 'logs.db')
550 xsrf_path
= os
.path
.join(storage_path
, 'xsrf')
552 search_index_path
= options
.search_indexes_path
or os
.path
.join(
553 storage_path
, 'search_indexes')
555 prospective_search_path
= options
.prospective_search_path
or os
.path
.join(
556 storage_path
, 'prospective-search')
558 blobstore_path
= options
.blobstore_path
or os
.path
.join(storage_path
,
561 if options
.clear_datastore
:
562 _clear_datastore_storage(datastore_path
)
564 if options
.clear_prospective_search
:
565 _clear_prospective_search_storage(prospective_search_path
)
567 if options
.clear_search_indexes
:
568 _clear_search_indexes_storage(search_index_path
)
570 if options
.auto_id_policy
==datastore_stub_util
.SEQUENTIAL
:
571 logging
.warn("--auto_id_policy='sequential' is deprecated. This option "
572 "will be removed in a future release.")
574 application_address
= '%s' % options
.host
575 if options
.port
and options
.port
!= 80:
576 application_address
+= ':' + str(options
.port
)
578 user_login_url
= '/%s?%s=%%s' % (login
.LOGIN_URL_RELATIVE
,
579 login
.CONTINUE_PARAM
)
580 user_logout_url
= '%s&%s=%s' % (user_login_url
, login
.ACTION_PARAM
,
583 if options
.datastore_consistency_policy
== 'time':
584 consistency
= datastore_stub_util
.TimeBasedHRConsistencyPolicy()
585 elif options
.datastore_consistency_policy
== 'random':
586 consistency
= datastore_stub_util
.PseudoRandomHRConsistencyPolicy()
587 elif options
.datastore_consistency_policy
== 'consistent':
588 consistency
= datastore_stub_util
.PseudoRandomHRConsistencyPolicy(1.0)
590 assert 0, ('unknown consistency policy: %r' %
591 options
.datastore_consistency_policy
)
593 api_server
.maybe_convert_datastore_file_stub_data_to_sqlite(
594 configuration
.app_id
, datastore_path
)
595 api_server
.setup_stubs(
596 request_data
=request_data
,
597 app_id
=configuration
.app_id
,
598 application_root
=configuration
.servers
[0].application_root
,
599 # The "trusted" flag is only relevant for Google administrative
601 trusted
=getattr(options
, 'trusted', False),
602 blobstore_path
=blobstore_path
,
603 datastore_path
=datastore_path
,
604 datastore_consistency
=consistency
,
605 datastore_require_indexes
=options
.require_indexes
,
606 datastore_auto_id_policy
=options
.auto_id_policy
,
607 images_host_prefix
='http://%s' % application_address
,
609 mail_smtp_host
=options
.smtp_host
,
610 mail_smtp_port
=options
.smtp_port
,
611 mail_smtp_user
=options
.smtp_user
,
612 mail_smtp_password
=options
.smtp_password
,
613 mail_enable_sendmail
=options
.enable_sendmail
,
614 mail_show_mail_body
=options
.show_mail_body
,
615 matcher_prospective_search_path
=prospective_search_path
,
616 search_index_path
=search_index_path
,
617 taskqueue_auto_run_tasks
=options
.enable_task_running
,
618 taskqueue_default_http_server
=application_address
,
619 user_login_url
=user_login_url
,
620 user_logout_url
=user_logout_url
)
622 # The APIServer must bind to localhost because that is what the runtime
624 apis
= api_server
.APIServer('localhost', options
.api_port
,
625 configuration
.app_id
)
627 self
._running
_servers
.append(apis
)
629 self
._running
_servers
.append(self
._dispatcher
)
630 self
._dispatcher
.start(apis
.port
, request_data
)
632 admin
= admin_server
.AdminServer(options
.admin_host
, options
.admin_port
,
633 self
._dispatcher
, configuration
, xsrf_path
)
635 self
._running
_servers
.append(admin
)
638 """Stops all running devappserver2 servers."""
639 while self
._running
_servers
:
640 self
._running
_servers
.pop().quit()
644 shutdown
.install_signal_handlers()
645 # The timezone must be set in the devappserver2 process rather than just in
646 # the runtime so printed log timestamps are consistent and the taskqueue stub
647 # expects the timezone to be UTC. The runtime inherits the environment.
648 os
.environ
['TZ'] = 'UTC'
649 if hasattr(time
, 'tzset'):
650 # time.tzet() should be called on Unix, but doesn't exist on Windows.
652 options
= PARSER
.parse_args()
653 dev_server
= DevelopmentServer()
655 dev_server
.start(options
)
656 shutdown
.wait_until_shutdown()
661 if __name__
== '__main__':