python/google/appengine/tools/devappserver2/devappserver2.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """The main entry point for the new development server."""
  18
  19
  20 import argparse
  21 import errno
  22 import getpass
  23 import itertools
  24 import logging
  25 import os
  26 import sys
  27 import tempfile
  28 import time
  29
  30 from google.appengine.api import appinfo
  31 from google.appengine.datastore import datastore_stub_util
  32 from google.appengine.tools import boolean_action
  33 from google.appengine.tools.devappserver2 import api_server
  34 from google.appengine.tools.devappserver2 import application_configuration
  35 from google.appengine.tools.devappserver2 import dispatcher
  36 from google.appengine.tools.devappserver2 import gcd_application
  37 from google.appengine.tools.devappserver2 import login
  38 from google.appengine.tools.devappserver2 import runtime_config_pb2
  39 from google.appengine.tools.devappserver2 import shutdown
  40 from google.appengine.tools.devappserver2 import update_checker
  41 from google.appengine.tools.devappserver2 import wsgi_request_info
  42 from google.appengine.tools.devappserver2.admin import admin_server
  43
  44 # Initialize logging early -- otherwise some library packages may
  45 # pre-empt our log formatting.  NOTE: the level is provisional; it may
  46 # be changed in main() based on the --debug flag.
  47 logging.basicConfig(
  48     level=logging.INFO,
  49     format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
  50
  51 # Valid choices for --log_level and their corresponding constants in
  52 # runtime_config_pb2.Config.stderr_log_level.
  53 _LOG_LEVEL_TO_RUNTIME_CONSTANT = {
  54     'debug': 0,
  55     'info': 1,
  56     'warning': 2,
  57     'error': 3,
  58     'critical': 4,
  59 }
  60
  61 # Valid choices for --dev_appserver_log_level and their corresponding Python
  62 # logging levels
  63 _LOG_LEVEL_TO_PYTHON_CONSTANT = {
  64     'debug': logging.DEBUG,
  65     'info': logging.INFO,
  66     'warning': logging.WARNING,
  67     'error': logging.ERROR,
  68     'critical': logging.CRITICAL,
  69 }
  70
  71 # The default encoding used by the production interpreter.
  72 _PROD_DEFAULT_ENCODING = 'ascii'
  73
  74
  75 def _generate_storage_paths(app_id):
  76   """Yield an infinite sequence of possible storage paths."""
  77   if sys.platform == 'win32':
  78     # The temp directory is per-user on Windows so there is no reason to add
  79     # the username to the generated directory name.
  80     user_format = ''
  81   else:
  82     try:
  83       user_name = getpass.getuser()
  84     except Exception:  # The possible set of exceptions is not documented.
  85       user_format = ''
  86     else:
  87       user_format = '.%s' % user_name
  88
  89   tempdir = tempfile.gettempdir()
  90   yield os.path.join(tempdir, 'appengine.%s%s' % (app_id, user_format))
  91   for i in itertools.count(1):
  92     yield os.path.join(tempdir, 'appengine.%s%s.%d' % (app_id, user_format, i))
  93
  94
  95 def _get_storage_path(path, app_id):
  96   """Returns a path to the directory where stub data can be stored."""
  97   _, _, app_id = app_id.replace(':', '_').rpartition('~')
  98   if path is None:
  99     for path in _generate_storage_paths(app_id):
 100       try:
 101         os.mkdir(path, 0700)
 102       except OSError, e:
 103         if e.errno == errno.EEXIST:
 104           # Check that the directory is only accessable by the current user to
 105           # protect against an attacker creating the directory in advance in
 106           # order to access any created files. Windows has per-user temporary
 107           # directories and st_mode does not include per-user permission
 108           # information so assume that it is safe.
 109           if sys.platform == 'win32' or (
 110               (os.stat(path).st_mode & 0777) == 0700 and os.path.isdir(path)):
 111             return path
 112           else:
 113             continue
 114         raise
 115       else:
 116         return path
 117   elif not os.path.exists(path):
 118     os.mkdir(path)
 119     return path
 120   elif not os.path.isdir(path):
 121     raise IOError('the given storage path %r is a file, a directory was '
 122                   'expected' % path)
 123   else:
 124     return path
 125
 126
 127 def _get_default_php_path():
 128   """Returns the path to the siloed php-cgi binary or None if not present."""
 129   default_php_executable_path = None
 130   if sys.platform == 'win32':
 131     default_php_executable_path = os.path.abspath(
 132         os.path.join(os.path.dirname(sys.argv[0]),
 133                      'php/php-5.4-Win32-VC9-x86/php-cgi.exe'))
 134   elif sys.platform == 'darwin':
 135     # The Cloud SDK uses symlinks in its packaging of the Mac Launcher.  First
 136     # try to find PHP relative to the apsolute path of this executable.  If that
 137     # doesn't work, try using the path without dereferencing all symlinks.
 138     base_paths = [os.path.realpath(sys.argv[0]), sys.argv[0]]
 139     for base_path in base_paths:
 140       default_php_executable_path = os.path.abspath(
 141           os.path.join(os.path.dirname(os.path.dirname(base_path)), 'php-cgi'))
 142       if os.path.exists(default_php_executable_path):
 143         break
 144
 145   if (default_php_executable_path and
 146       os.path.exists(default_php_executable_path)):
 147     return default_php_executable_path
 148   return None
 149
 150
 151 class PortParser(object):
 152   """A parser for ints that represent ports."""
 153
 154   def __init__(self, allow_port_zero=True):
 155     self._min_port = 0 if allow_port_zero else 1
 156
 157   def __call__(self, value):
 158     try:
 159       port = int(value)
 160     except ValueError:
 161       raise argparse.ArgumentTypeError('Invalid port: %r' % value)
 162     if port < self._min_port or port >= (1 << 16):
 163       raise argparse.ArgumentTypeError('Invalid port: %d' % port)
 164     return port
 165
 166
 167 def parse_per_module_option(
 168     value, value_type, value_predicate,
 169     single_bad_type_error, single_bad_predicate_error,
 170     multiple_bad_type_error, multiple_bad_predicate_error,
 171     multiple_duplicate_module_error):
 172   """Parses command line options that may be specified per-module.
 173
 174   Args:
 175     value: A str containing the flag value to parse. Two formats are supported:
 176         1. A universal value (may not contain a colon as that is use to
 177            indicate a per-module value).
 178         2. Per-module values. One or more comma separated module-value pairs.
 179            Each pair is a module_name:value. An empty module-name is shorthand
 180            for "default" to match how not specifying a module name in the yaml
 181            is the same as specifying "module: default".
 182     value_type: a callable that converts the string representation of the value
 183         to the actual value. Should raise ValueError if the string can not
 184         be converted.
 185     value_predicate: a predicate to call on the converted value to validate
 186         the converted value. Use "lambda _: True" if all values are valid.
 187     single_bad_type_error: the message to use if a universal value is provided
 188         and value_type throws a ValueError. The message must consume a single
 189         format parameter (the provided value).
 190     single_bad_predicate_error: the message to use if a universal value is
 191         provided and value_predicate returns False. The message does not
 192         get any format parameters.
 193     multiple_bad_type_error: the message to use if a per-module value
 194         either does not have two values separated by a single colon or if
 195         value_types throws a ValueError on the second string. The message must
 196         consume a single format parameter (the module_name:value pair).
 197     multiple_bad_predicate_error: the message to use if a per-module value if
 198         value_predicate returns False. The message must consume a single format
 199         parameter (the module name).
 200     multiple_duplicate_module_error: the message to use if the same module is
 201         repeated. The message must consume a single formater parameter (the
 202         module name).
 203
 204   Returns:
 205     Either a single value of value_type for universal values or a dict of
 206     str->value_type for per-module values.
 207
 208   Raises:
 209     argparse.ArgumentTypeError: the value is invalid.
 210   """
 211   if ':' not in value:
 212     try:
 213       single_value = value_type(value)
 214     except ValueError:
 215       raise argparse.ArgumentTypeError(single_bad_type_error % value)
 216     else:
 217       if not value_predicate(single_value):
 218         raise argparse.ArgumentTypeError(single_bad_predicate_error)
 219       return single_value
 220   else:
 221     module_to_value = {}
 222     for module_value in value.split(','):
 223       try:
 224         module_name, single_value = module_value.split(':')
 225         single_value = value_type(single_value)
 226       except ValueError:
 227         raise argparse.ArgumentTypeError(multiple_bad_type_error % module_value)
 228       else:
 229         module_name = module_name.strip()
 230         if not module_name:
 231           module_name = appinfo.DEFAULT_MODULE
 232         if module_name in module_to_value:
 233           raise argparse.ArgumentTypeError(
 234               multiple_duplicate_module_error % module_name)
 235         if not value_predicate(single_value):
 236           raise argparse.ArgumentTypeError(
 237               multiple_bad_predicate_error % module_name)
 238         module_to_value[module_name] = single_value
 239     return module_to_value
 240
 241
 242 def parse_max_module_instances(value):
 243   """Returns the parsed value for the --max_module_instances flag.
 244
 245   Args:
 246     value: A str containing the flag value for parse. The format should follow
 247         one of the following examples:
 248           1. "5" - All modules are limited to 5 instances.
 249           2. "default:3,backend:20" - The default module can have 3 instances,
 250              "backend" can have 20 instances and all other modules are
 251               unaffected. An empty name (i.e. ":3") is shorthand for default
 252               to match how not specifying a module name in the yaml is the
 253               same as specifying "module: default".
 254   Returns:
 255     The parsed value of the max_module_instances flag. May either be an int
 256     (for values of the form "5") or a dict of str->int (for values of the
 257     form "default:3,backend:20").
 258
 259   Raises:
 260     argparse.ArgumentTypeError: the value is invalid.
 261   """
 262   return parse_per_module_option(
 263       value, int, lambda instances: instances > 0,
 264       'Invalid max instance count: %r',
 265       'Max instance count must be greater than zero',
 266       'Expected "module:max_instance_count": %r',
 267       'Max instance count for module %s must be greater than zero',
 268       'Duplicate max instance count for module %s')
 269
 270
 271 def parse_threadsafe_override(value):
 272   """Returns the parsed value for the --threadsafe_override flag.
 273
 274   Args:
 275     value: A str containing the flag value for parse. The format should follow
 276         one of the following examples:
 277           1. "False" - All modules override the YAML threadsafe configuration
 278              as if the YAML contained False.
 279           2. "default:False,backend:True" - The default module overrides the
 280              YAML threadsafe configuration as if the YAML contained False, the
 281              "backend" module overrides with a value of True and all other
 282              modules use the value in the YAML file. An empty name (i.e.
 283              ":True") is shorthand for default to match how not specifying a
 284              module name in the yaml is the same as specifying
 285              "module: default".
 286   Returns:
 287     The parsed value of the threadsafe_override flag. May either be a bool
 288     (for values of the form "False") or a dict of str->bool (for values of the
 289     form "default:False,backend:True").
 290
 291   Raises:
 292     argparse.ArgumentTypeError: the value is invalid.
 293   """
 294   return parse_per_module_option(
 295       value, boolean_action.BooleanParse, lambda _: True,
 296       'Invalid threadsafe override: %r',
 297       None,
 298       'Expected "module:threadsafe_override": %r',
 299       None,
 300       'Duplicate threadsafe override value for module %s')
 301
 302
 303 def parse_path(value):
 304   """Returns the given path with ~ and environment variables expanded."""
 305   return os.path.expanduser(os.path.expandvars(value))
 306
 307
 308 def create_command_line_parser():
 309   """Returns an argparse.ArgumentParser to parse command line arguments."""
 310   # TODO: Add more robust argument validation. Consider what flags
 311   # are actually needed.
 312
 313   parser = argparse.ArgumentParser(
 314       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 315   arg_name = 'yaml_path'
 316   arg_help = 'Path to a yaml file, or a directory containing yaml files'
 317   if application_configuration.java_supported():
 318     arg_name = 'yaml_or_war_path'
 319     arg_help += ', or a directory containing WEB-INF/web.xml'
 320   parser.add_argument(
 321       'config_paths', metavar=arg_name, nargs='+', help=arg_help)
 322
 323   common_group = parser.add_argument_group('Common')
 324   common_group.add_argument(
 325       '-A', '--application', action='store', dest='app_id',
 326       help='Set the application, overriding the application value from the '
 327       'app.yaml file.')
 328   common_group.add_argument(
 329       '--host', default='localhost',
 330       help='host name to which application modules should bind')
 331   common_group.add_argument(
 332       '--port', type=PortParser(), default=8080,
 333       help='lowest port to which application modules should bind')
 334   common_group.add_argument(
 335       '--admin_host', default='localhost',
 336       help='host name to which the admin server should bind')
 337   common_group.add_argument(
 338       '--admin_port', type=PortParser(), default=8000,
 339       help='port to which the admin server should bind')
 340   common_group.add_argument(
 341       '--auth_domain', default='gmail.com',
 342       help='name of the authorization domain to use')
 343   common_group.add_argument(
 344       '--storage_path', metavar='PATH',
 345       type=parse_path,
 346       help='path to the data (datastore, blobstore, etc.) associated with the '
 347       'application.')
 348   common_group.add_argument(
 349       '--log_level', default='info',
 350       choices=_LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(),
 351       help='the log level below which logging messages generated by '
 352       'application code will not be displayed on the console')
 353   common_group.add_argument(
 354       '--max_module_instances',
 355       type=parse_max_module_instances,
 356       help='the maximum number of runtime instances that can be started for a '
 357       'particular module - the value can be an integer, in what case all '
 358       'modules are limited to that number of instances or a comma-seperated '
 359       'list of module:max_instances e.g. "default:5,backend:3"')
 360   common_group.add_argument(
 361       '--use_mtime_file_watcher',
 362       action=boolean_action.BooleanAction,
 363       const=True,
 364       default=False,
 365       help='use mtime polling for detecting source code changes - useful if '
 366       'modifying code from a remote machine using a distributed file system')
 367   common_group.add_argument(
 368       '--threadsafe_override',
 369       type=parse_threadsafe_override,
 370       help='override the application\'s threadsafe configuration - the value '
 371       'can be a boolean, in which case all modules threadsafe setting will '
 372       'be overridden or a comma-separated list of module:threadsafe_override '
 373       'e.g. "default:False,backend:True"')
 374   common_group.add_argument('--docker_daemon_url', help=argparse.SUPPRESS)
 375
 376   # PHP
 377   php_group = parser.add_argument_group('PHP')
 378   php_group.add_argument('--php_executable_path', metavar='PATH',
 379                          type=parse_path,
 380                          default=_get_default_php_path(),
 381                          help='path to the PHP executable')
 382   php_group.add_argument('--php_remote_debugging',
 383                          action=boolean_action.BooleanAction,
 384                          const=True,
 385                          default=False,
 386                          help='enable XDebug remote debugging')
 387
 388   # Dart
 389   dart_group = parser.add_argument_group('Dart')
 390   dart_group.add_argument('--dart_sdk', help=argparse.SUPPRESS)
 391   dart_group.add_argument('--dart_dev_mode',
 392                           choices=['dev', 'deploy'],
 393                           help=argparse.SUPPRESS)
 394   dart_group.add_argument('--dart_pub_serve_host', help=argparse.SUPPRESS)
 395   dart_group.add_argument('--dart_pub_serve_port',
 396                           type=PortParser(), help=argparse.SUPPRESS)
 397
 398   # App Identity
 399   appidentity_group = parser.add_argument_group('Application Identity')
 400   appidentity_group.add_argument(
 401       '--appidentity_email_address',
 402       help='email address associated with a service account that has a '
 403       'downloadable key. May be None for no local application identity.')
 404   appidentity_group.add_argument(
 405       '--appidentity_private_key_path',
 406       help='path to private key file associated with service account '
 407       '(.pem format). Must be set if appidentity_email_address is set.')
 408
 409   # Python
 410   python_group = parser.add_argument_group('Python')
 411   python_group.add_argument(
 412       '--python_startup_script',
 413       help='the script to run at the startup of new Python runtime instances '
 414       '(useful for tools such as debuggers.')
 415   python_group.add_argument(
 416       '--python_startup_args',
 417       help='the arguments made available to the script specified in '
 418       '--python_startup_script.')
 419
 420   # Java
 421   java_group = parser.add_argument_group('Java')
 422   java_group.add_argument(
 423       '--jvm_flag', action='append',
 424       help='additional arguments to pass to the java command when launching '
 425       'an instance of the app. May be specified more than once. Example: '
 426       '--jvm_flag=-Xmx1024m --jvm_flag=-Xms256m')
 427
 428   # Blobstore
 429   blobstore_group = parser.add_argument_group('Blobstore API')
 430   blobstore_group.add_argument(
 431       '--blobstore_path',
 432       type=parse_path,
 433       help='path to directory used to store blob contents '
 434       '(defaults to a subdirectory of --storage_path if not set)',
 435       default=None)
 436
 437   # Cloud SQL
 438   cloud_sql_group = parser.add_argument_group('Cloud SQL')
 439   cloud_sql_group.add_argument(
 440       '--mysql_host',
 441       default='localhost',
 442       help='host name of a running MySQL server used for simulated Google '
 443       'Cloud SQL storage')
 444   cloud_sql_group.add_argument(
 445       '--mysql_port', type=PortParser(allow_port_zero=False),
 446       default=3306,
 447       help='port number of a running MySQL server used for simulated Google '
 448       'Cloud SQL storage')
 449   cloud_sql_group.add_argument(
 450       '--mysql_user',
 451       default='',
 452       help='username to use when connecting to the MySQL server specified in '
 453       '--mysql_host and --mysql_port or --mysql_socket')
 454   cloud_sql_group.add_argument(
 455       '--mysql_password',
 456       default='',
 457       help='password to use when connecting to the MySQL server specified in '
 458       '--mysql_host and --mysql_port or --mysql_socket')
 459   cloud_sql_group.add_argument(
 460       '--mysql_socket',
 461       help='path to a Unix socket file to use when connecting to a running '
 462       'MySQL server used for simulated Google Cloud SQL storage')
 463
 464   # Datastore
 465   datastore_group = parser.add_argument_group('Datastore API')
 466   datastore_group.add_argument(
 467       '--datastore_path',
 468       type=parse_path,
 469       default=None,
 470       help='path to a file used to store datastore contents '
 471       '(defaults to a file in --storage_path if not set)',)
 472   datastore_group.add_argument('--clear_datastore',
 473                                action=boolean_action.BooleanAction,
 474                                const=True,
 475                                default=False,
 476                                help='clear the datastore on startup')
 477   datastore_group.add_argument(
 478       '--datastore_consistency_policy',
 479       default='time',
 480       choices=['consistent', 'random', 'time'],
 481       help='the policy to apply when deciding whether a datastore write should '
 482       'appear in global queries')
 483   datastore_group.add_argument(
 484       '--require_indexes',
 485       action=boolean_action.BooleanAction,
 486       const=True,
 487       default=False,
 488       help='generate an error on datastore queries that '
 489       'requires a composite index not found in index.yaml')
 490   datastore_group.add_argument(
 491       '--auto_id_policy',
 492       default=datastore_stub_util.SCATTERED,
 493       choices=[datastore_stub_util.SEQUENTIAL,
 494                datastore_stub_util.SCATTERED],
 495       help='the type of sequence from which the datastore stub '
 496       'assigns automatic IDs. NOTE: Sequential IDs are '
 497       'deprecated. This flag will be removed in a future '
 498       'release. Please do not rely on sequential IDs in your '
 499       'tests.')
 500   datastore_group.add_argument(
 501       '--enable_cloud_datastore',
 502       action=boolean_action.BooleanAction,
 503       const=True,
 504       default=False,
 505       help=argparse.SUPPRESS #'enable the Google Cloud Datastore API.'
 506       )
 507
 508   # Logs
 509   logs_group = parser.add_argument_group('Logs API')
 510   logs_group.add_argument(
 511       '--logs_path', default=None,
 512       help='path to a file used to store request logs (defaults to a file in '
 513       '--storage_path if not set)',)
 514
 515   # Mail
 516   mail_group = parser.add_argument_group('Mail API')
 517   mail_group.add_argument(
 518       '--show_mail_body',
 519       action=boolean_action.BooleanAction,
 520       const=True,
 521       default=False,
 522       help='logs the contents of e-mails sent using the Mail API')
 523   mail_group.add_argument(
 524       '--enable_sendmail',
 525       action=boolean_action.BooleanAction,
 526       const=True,
 527       default=False,
 528       help='use the "sendmail" tool to transmit e-mail sent '
 529       'using the Mail API (ignored if --smtp_host is set)')
 530   mail_group.add_argument(
 531       '--smtp_host', default='',
 532       help='host name of an SMTP server to use to transmit '
 533       'e-mail sent using the Mail API')
 534   mail_group.add_argument(
 535       '--smtp_port', default=25,
 536       type=PortParser(allow_port_zero=False),
 537       help='port number of an SMTP server to use to transmit '
 538       'e-mail sent using the Mail API (ignored if --smtp_host '
 539       'is not set)')
 540   mail_group.add_argument(
 541       '--smtp_user', default='',
 542       help='username to use when connecting to the SMTP server '
 543       'specified in --smtp_host and --smtp_port')
 544   mail_group.add_argument(
 545       '--smtp_password', default='',
 546       help='password to use when connecting to the SMTP server '
 547       'specified in --smtp_host and --smtp_port')
 548   mail_group.add_argument(
 549       '--smtp_allow_tls',
 550       action=boolean_action.BooleanAction,
 551       const=True,
 552       default=True,
 553       help='Allow TLS to be used when the SMTP server announces TLS support '
 554       '(ignored if --smtp_host is not set)')
 555
 556   # Matcher
 557   prospective_search_group = parser.add_argument_group('Prospective Search API')
 558   prospective_search_group.add_argument(
 559       '--prospective_search_path', default=None,
 560       type=parse_path,
 561       help='path to a file used to store the prospective '
 562       'search subscription index (defaults to a file in '
 563       '--storage_path if not set)')
 564   prospective_search_group.add_argument(
 565       '--clear_prospective_search',
 566       action=boolean_action.BooleanAction,
 567       const=True,
 568       default=False,
 569       help='clear the prospective search subscription index')
 570
 571   # Search
 572   search_group = parser.add_argument_group('Search API')
 573   search_group.add_argument(
 574       '--search_indexes_path', default=None,
 575       type=parse_path,
 576       help='path to a file used to store search indexes '
 577       '(defaults to a file in --storage_path if not set)',)
 578   search_group.add_argument(
 579       '--clear_search_indexes',
 580       action=boolean_action.BooleanAction,
 581       const=True,
 582       default=False,
 583       help='clear the search indexes')
 584
 585   # Taskqueue
 586   taskqueue_group = parser.add_argument_group('Task Queue API')
 587   taskqueue_group.add_argument(
 588       '--enable_task_running',
 589       action=boolean_action.BooleanAction,
 590       const=True,
 591       default=True,
 592       help='run "push" tasks created using the taskqueue API automatically')
 593
 594   # Misc
 595   misc_group = parser.add_argument_group('Miscellaneous')
 596   misc_group.add_argument(
 597       '--allow_skipped_files',
 598       action=boolean_action.BooleanAction,
 599       const=True,
 600       default=False,
 601       help='make files specified in the app.yaml "skip_files" or "static" '
 602       'handles readable by the application.')
 603   # No help to avoid lengthening help message for rarely used feature:
 604   # host name to which the server for API calls should bind.
 605   misc_group.add_argument(
 606       '--api_host', default='localhost',
 607       help=argparse.SUPPRESS)
 608   misc_group.add_argument(
 609       '--api_port', type=PortParser(), default=0,
 610       help='port to which the server for API calls should bind')
 611   misc_group.add_argument(
 612       '--automatic_restart',
 613       action=boolean_action.BooleanAction,
 614       const=True,
 615       default=True,
 616       help=('restart instances automatically when files relevant to their '
 617             'module are changed'))
 618   misc_group.add_argument(
 619       '--dev_appserver_log_level', default='info',
 620       choices=_LOG_LEVEL_TO_PYTHON_CONSTANT.keys(),
 621       help='the log level below which logging messages generated by '
 622       'the development server will not be displayed on the console (this '
 623       'flag is more useful for diagnosing problems in dev_appserver.py rather '
 624       'than in application code)')
 625   misc_group.add_argument(
 626       '--skip_sdk_update_check',
 627       action=boolean_action.BooleanAction,
 628       const=True,
 629       default=False,
 630       help='skip checking for SDK updates (if false, use .appcfg_nag to '
 631       'decide)')
 632   misc_group.add_argument(
 633       '--default_gcs_bucket_name', default=None,
 634       help='default Google Cloud Storgage bucket name')
 635
 636
 637   return parser
 638
 639 PARSER = create_command_line_parser()
 640
 641
 642 def _clear_datastore_storage(datastore_path):
 643   """Delete the datastore storage file at the given path."""
 644   # lexists() returns True for broken symlinks, where exists() returns False.
 645   if os.path.lexists(datastore_path):
 646     try:
 647       os.remove(datastore_path)
 648     except OSError, e:
 649       logging.warning('Failed to remove datastore file %r: %s',
 650                       datastore_path,
 651                       e)
 652
 653
 654 def _clear_prospective_search_storage(prospective_search_path):
 655   """Delete the perspective search storage file at the given path."""
 656   # lexists() returns True for broken symlinks, where exists() returns False.
 657   if os.path.lexists(prospective_search_path):
 658     try:
 659       os.remove(prospective_search_path)
 660     except OSError, e:
 661       logging.warning('Failed to remove prospective search file %r: %s',
 662                       prospective_search_path,
 663                       e)
 664
 665
 666 def _clear_search_indexes_storage(search_index_path):
 667   """Delete the search indexes storage file at the given path."""
 668   # lexists() returns True for broken symlinks, where exists() returns False.
 669   if os.path.lexists(search_index_path):
 670     try:
 671       os.remove(search_index_path)
 672     except OSError, e:
 673       logging.warning('Failed to remove search indexes file %r: %s',
 674                       search_index_path,
 675                       e)
 676
 677
 678 def _setup_environ(app_id):
 679   """Sets up the os.environ dictionary for the front-end server and API server.
 680
 681   This function should only be called once.
 682
 683   Args:
 684     app_id: The id of the application.
 685   """
 686   os.environ['APPLICATION_ID'] = app_id
 687
 688
 689 class DevelopmentServer(object):
 690   """Encapsulates the logic for the development server.
 691
 692   Only a single instance of the class may be created per process. See
 693   _setup_environ.
 694   """
 695
 696   def __init__(self):
 697     # A list of servers that are currently running.
 698     self._running_modules = []
 699     self._module_to_port = {}
 700     self._dispatcher = None
 701
 702   def module_to_address(self, module_name, instance=None):
 703     """Returns the address of a module."""
 704
 705     if module_name is None:
 706       return self._dispatcher.dispatch_address
 707     return self._dispatcher.get_hostname(
 708         module_name,
 709         self._dispatcher.get_default_version(module_name),
 710         instance)
 711
 712   def start(self, options):
 713     """Start devappserver2 servers based on the provided command line arguments.
 714
 715     Args:
 716       options: An argparse.Namespace containing the command line arguments.
 717     """
 718     logging.getLogger().setLevel(
 719         _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
 720
 721     configuration = application_configuration.ApplicationConfiguration(
 722         options.config_paths, options.app_id)
 723
 724     if options.enable_cloud_datastore:
 725       # This requires the oauth server stub to return that the logged in user
 726       # is in fact an admin.
 727       os.environ['OAUTH_IS_ADMIN'] = '1'
 728       gcd_module = application_configuration.ModuleConfiguration(
 729           gcd_application.generate_gcd_app(configuration.app_id.split('~')[1]))
 730       configuration.modules.append(gcd_module)
 731
 732     if options.skip_sdk_update_check:
 733       logging.info('Skipping SDK update check.')
 734     else:
 735       update_checker.check_for_updates(configuration)
 736
 737     # There is no good way to set the default encoding from application code
 738     # (it needs to be done during interpreter initialization in site.py or
 739     # sitecustomize.py) so just warn developers if they have a different
 740     # encoding than production.
 741     if sys.getdefaultencoding() != _PROD_DEFAULT_ENCODING:
 742       logging.warning(
 743           'The default encoding of your local Python interpreter is set to %r '
 744           'while App Engine\'s production environment uses %r; as a result '
 745           'your code may behave differently when deployed.',
 746           sys.getdefaultencoding(), _PROD_DEFAULT_ENCODING)
 747
 748     if options.port == 0:
 749       logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
 750                    '--port=0')
 751
 752     _setup_environ(configuration.app_id)
 753
 754     self._dispatcher = dispatcher.Dispatcher(
 755         configuration,
 756         options.host,
 757         options.port,
 758         options.auth_domain,
 759         _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
 760         self._create_php_config(options),
 761         self._create_python_config(options),
 762         self._create_java_config(options),
 763         self._create_cloud_sql_config(options),
 764         self._create_vm_config(options),
 765         self._create_module_to_setting(options.max_module_instances,
 766                                        configuration, '--max_module_instances'),
 767         options.use_mtime_file_watcher,
 768         options.automatic_restart,
 769         options.allow_skipped_files,
 770         self._create_module_to_setting(options.threadsafe_override,
 771                                        configuration, '--threadsafe_override'))
 772
 773     request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
 774     storage_path = _get_storage_path(options.storage_path, configuration.app_id)
 775
 776     apis = self._create_api_server(
 777         request_data, storage_path, options, configuration)
 778     apis.start()
 779     self._running_modules.append(apis)
 780
 781     self._dispatcher.start(options.api_host, apis.port, request_data)
 782
 783     xsrf_path = os.path.join(storage_path, 'xsrf')
 784     admin = admin_server.AdminServer(options.admin_host, options.admin_port,
 785                                      self._dispatcher, configuration, xsrf_path)
 786     admin.start()
 787     self._running_modules.append(admin)
 788
 789   def stop(self):
 790     """Stops all running devappserver2 modules."""
 791     while self._running_modules:
 792       self._running_modules.pop().quit()
 793     if self._dispatcher:
 794       self._dispatcher.quit()
 795
 796   @staticmethod
 797   def _create_api_server(request_data, storage_path, options, configuration):
 798     datastore_path = options.datastore_path or os.path.join(storage_path,
 799                                                             'datastore.db')
 800     logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
 801
 802     search_index_path = options.search_indexes_path or os.path.join(
 803         storage_path, 'search_indexes')
 804
 805     prospective_search_path = options.prospective_search_path or os.path.join(
 806         storage_path, 'prospective-search')
 807
 808     blobstore_path = options.blobstore_path or os.path.join(storage_path,
 809                                                             'blobs')
 810
 811     if options.clear_datastore:
 812       _clear_datastore_storage(datastore_path)
 813
 814     if options.clear_prospective_search:
 815       _clear_prospective_search_storage(prospective_search_path)
 816
 817     if options.clear_search_indexes:
 818       _clear_search_indexes_storage(search_index_path)
 819
 820     if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
 821       logging.warn("--auto_id_policy='sequential' is deprecated. This option "
 822                    "will be removed in a future release.")
 823
 824     application_address = '%s' % options.host
 825     if options.port and options.port != 80:
 826       application_address += ':' + str(options.port)
 827
 828     user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
 829                                      login.CONTINUE_PARAM)
 830     user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
 831                                     login.LOGOUT_ACTION)
 832
 833     if options.datastore_consistency_policy == 'time':
 834       consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
 835     elif options.datastore_consistency_policy == 'random':
 836       consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
 837     elif options.datastore_consistency_policy == 'consistent':
 838       consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
 839     else:
 840       assert 0, ('unknown consistency policy: %r' %
 841                  options.datastore_consistency_policy)
 842
 843     api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
 844         configuration.app_id, datastore_path)
 845     api_server.setup_stubs(
 846         request_data=request_data,
 847         app_id=configuration.app_id,
 848         application_root=configuration.modules[0].application_root,
 849         # The "trusted" flag is only relevant for Google administrative
 850         # applications.
 851         trusted=getattr(options, 'trusted', False),
 852         appidentity_email_address=options.appidentity_email_address,
 853         appidentity_private_key_path=os.path.abspath(
 854             options.appidentity_private_key_path)
 855         if options.appidentity_private_key_path else None,
 856         blobstore_path=blobstore_path,
 857         datastore_path=datastore_path,
 858         datastore_consistency=consistency,
 859         datastore_require_indexes=options.require_indexes,
 860         datastore_auto_id_policy=options.auto_id_policy,
 861         images_host_prefix='http://%s' % application_address,
 862         logs_path=logs_path,
 863         mail_smtp_host=options.smtp_host,
 864         mail_smtp_port=options.smtp_port,
 865         mail_smtp_user=options.smtp_user,
 866         mail_smtp_password=options.smtp_password,
 867         mail_enable_sendmail=options.enable_sendmail,
 868         mail_show_mail_body=options.show_mail_body,
 869         mail_allow_tls=options.smtp_allow_tls,
 870         matcher_prospective_search_path=prospective_search_path,
 871         search_index_path=search_index_path,
 872         taskqueue_auto_run_tasks=options.enable_task_running,
 873         taskqueue_default_http_server=application_address,
 874         user_login_url=user_login_url,
 875         user_logout_url=user_logout_url,
 876         default_gcs_bucket_name=options.default_gcs_bucket_name)
 877
 878     return api_server.APIServer(options.api_host, options.api_port,
 879                                 configuration.app_id)
 880
 881   @staticmethod
 882   def _create_php_config(options):
 883     php_config = runtime_config_pb2.PhpConfig()
 884     if options.php_executable_path:
 885       php_config.php_executable_path = os.path.abspath(
 886           options.php_executable_path)
 887     php_config.enable_debugger = options.php_remote_debugging
 888     return php_config
 889
 890   @staticmethod
 891   def _create_python_config(options):
 892     python_config = runtime_config_pb2.PythonConfig()
 893     if options.python_startup_script:
 894       python_config.startup_script = os.path.abspath(
 895           options.python_startup_script)
 896       if options.python_startup_args:
 897         python_config.startup_args = options.python_startup_args
 898     return python_config
 899
 900   @staticmethod
 901   def _create_java_config(options):
 902     java_config = runtime_config_pb2.JavaConfig()
 903     if options.jvm_flag:
 904       java_config.jvm_args.extend(options.jvm_flag)
 905     return java_config
 906
 907   @staticmethod
 908   def _create_cloud_sql_config(options):
 909     cloud_sql_config = runtime_config_pb2.CloudSQL()
 910     cloud_sql_config.mysql_host = options.mysql_host
 911     cloud_sql_config.mysql_port = options.mysql_port
 912     cloud_sql_config.mysql_user = options.mysql_user
 913     cloud_sql_config.mysql_password = options.mysql_password
 914     if options.mysql_socket:
 915       cloud_sql_config.mysql_socket = options.mysql_socket
 916     return cloud_sql_config
 917
 918   @staticmethod
 919   def _create_vm_config(options):
 920     vm_config = runtime_config_pb2.VMConfig()
 921     if options.docker_daemon_url:
 922       vm_config.docker_daemon_url = options.docker_daemon_url
 923     if options.dart_sdk:
 924       vm_config.dart_config.dart_sdk = os.path.abspath(options.dart_sdk)
 925     if options.dart_dev_mode:
 926       vm_config.dart_config.dart_dev_mode = options.dart_dev_mode
 927     if options.dart_pub_serve_host:
 928       vm_config.dart_config.dart_pub_serve_host = options.dart_pub_serve_host
 929     if options.dart_pub_serve_port:
 930       vm_config.dart_config.dart_pub_serve_port = options.dart_pub_serve_port
 931     return vm_config
 932
 933   @staticmethod
 934   def _create_module_to_setting(setting, configuration, option):
 935     """Create a per-module dictionary configuration.
 936
 937     Creates a dictionary that maps a module name to a configuration
 938     setting. Used in conjunction with parse_per_module_option.
 939
 940     Args:
 941       setting: a value that can be None, a dict of str->type or a single value.
 942       configuration: an ApplicationConfiguration object.
 943       option: the option name the setting came from.
 944
 945     Returns:
 946       A dict of str->type.
 947     """
 948     if setting is None:
 949       return {}
 950
 951     module_names = [module_configuration.module_name
 952                     for module_configuration in configuration.modules]
 953     if isinstance(setting, dict):
 954       # Warn and remove a setting if the module name is unknown.
 955       module_to_setting = {}
 956       for module_name, value in setting.items():
 957         if module_name in module_names:
 958           module_to_setting[module_name] = value
 959         else:
 960           logging.warning('Unknown module %r for %r', module_name, option)
 961       return module_to_setting
 962
 963     # Create a dict with an entry for every module.
 964     return {module_name: setting for module_name in module_names}
 965
 966
 967 def main():
 968   shutdown.install_signal_handlers()
 969   # The timezone must be set in the devappserver2 process rather than just in
 970   # the runtime so printed log timestamps are consistent and the taskqueue stub
 971   # expects the timezone to be UTC. The runtime inherits the environment.
 972   os.environ['TZ'] = 'UTC'
 973   if hasattr(time, 'tzset'):
 974     # time.tzet() should be called on Unix, but doesn't exist on Windows.
 975     time.tzset()
 976   options = PARSER.parse_args()
 977   dev_server = DevelopmentServer()
 978   try:
 979     dev_server.start(options)
 980     shutdown.wait_until_shutdown()
 981   finally:
 982     dev_server.stop()
 983
 984
 985 if __name__ == '__main__':
 986   main()