python/google/appengine/tools/devappserver2/devappserver2.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """The main entry point for the new development server."""
  18
  19
  20 import argparse
  21 import errno
  22 import getpass
  23 import itertools
  24 import logging
  25 import os
  26 import sys
  27 import tempfile
  28 import time
  29
  30 from google.appengine.api import appinfo
  31 from google.appengine.datastore import datastore_stub_util
  32 from google.appengine.tools import boolean_action
  33 from google.appengine.tools.devappserver2.admin import admin_server
  34 from google.appengine.tools.devappserver2 import api_server
  35 from google.appengine.tools.devappserver2 import application_configuration
  36 from google.appengine.tools.devappserver2 import dispatcher
  37 from google.appengine.tools.devappserver2 import login
  38 from google.appengine.tools.devappserver2 import runtime_config_pb2
  39 from google.appengine.tools.devappserver2 import shutdown
  40 from google.appengine.tools.devappserver2 import update_checker
  41 from google.appengine.tools.devappserver2 import wsgi_request_info
  42
  43 # Initialize logging early -- otherwise some library packages may
  44 # pre-empt our log formatting.  NOTE: the level is provisional; it may
  45 # be changed in main() based on the --debug flag.
  46 logging.basicConfig(
  47     level=logging.INFO,
  48     format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
  49
  50 # Valid choices for --log_level and their corresponding constants in
  51 # runtime_config_pb2.Config.stderr_log_level.
  52 _LOG_LEVEL_TO_RUNTIME_CONSTANT = {
  53     'debug': 0,
  54     'info': 1,
  55     'warning': 2,
  56     'error': 3,
  57     'critical': 4,
  58 }
  59
  60 # Valid choices for --dev_appserver_log_level and their corresponding Python
  61 # logging levels
  62 _LOG_LEVEL_TO_PYTHON_CONSTANT = {
  63     'debug': logging.DEBUG,
  64     'info': logging.INFO,
  65     'warning': logging.WARNING,
  66     'error': logging.ERROR,
  67     'critical': logging.CRITICAL,
  68 }
  69
  70 # The default encoding used by the production interpreter.
  71 _PROD_DEFAULT_ENCODING = 'ascii'
  72
  73
  74 def _generate_storage_paths(app_id):
  75   """Yield an infinite sequence of possible storage paths."""
  76   if sys.platform == 'win32':
  77     # The temp directory is per-user on Windows so there is no reason to add
  78     # the username to the generated directory name.
  79     user_format = ''
  80   else:
  81     try:
  82       user_name = getpass.getuser()
  83     except Exception:  # The possible set of exceptions is not documented.
  84       user_format = ''
  85     else:
  86       user_format = '.%s' % user_name
  87
  88   tempdir = tempfile.gettempdir()
  89   yield os.path.join(tempdir, 'appengine.%s%s' % (app_id, user_format))
  90   for i in itertools.count(1):
  91     yield os.path.join(tempdir, 'appengine.%s%s.%d' % (app_id, user_format, i))
  92
  93
  94 def _get_storage_path(path, app_id):
  95   """Returns a path to the directory where stub data can be stored."""
  96   _, _, app_id = app_id.replace(':', '_').rpartition('~')
  97   if path is None:
  98     for path in _generate_storage_paths(app_id):
  99       try:
 100         os.mkdir(path, 0700)
 101       except OSError, e:
 102         if e.errno == errno.EEXIST:
 103           # Check that the directory is only accessable by the current user to
 104           # protect against an attacker creating the directory in advance in
 105           # order to access any created files. Windows has per-user temporary
 106           # directories and st_mode does not include per-user permission
 107           # information so assume that it is safe.
 108           if sys.platform == 'win32' or (
 109               (os.stat(path).st_mode & 0777) == 0700 and os.path.isdir(path)):
 110             return path
 111           else:
 112             continue
 113         raise
 114       else:
 115         return path
 116   elif not os.path.exists(path):
 117     os.mkdir(path)
 118     return path
 119   elif not os.path.isdir(path):
 120     raise IOError('the given storage path %r is a file, a directory was '
 121                   'expected' % path)
 122   else:
 123     return path
 124
 125
 126 def _get_default_php_path():
 127   """Returns the path to the siloed php-cgi binary or None if not present."""
 128   default_php_executable_path = None
 129   if sys.platform == 'win32':
 130     default_php_executable_path = os.path.abspath(
 131         os.path.join(os.path.dirname(sys.argv[0]),
 132                      'php/php-5.4-Win32-VC9-x86/php-cgi.exe'))
 133   elif sys.platform == 'darwin':
 134     default_php_executable_path = os.path.abspath(
 135         os.path.join(
 136             os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))),
 137             'php-cgi'))
 138
 139   if (default_php_executable_path and
 140       os.path.exists(default_php_executable_path)):
 141     return default_php_executable_path
 142   return None
 143
 144
 145 def _generate_gcd_app(app_id):
 146   """Generates an app in tmp for a cloud datastore implementation."""
 147   if sys.platform == 'win32':
 148     # The temp directory is per-user on Windows so there is no reason to add
 149     # the username to the generated directory name.
 150     user_format = ''
 151   else:
 152     try:
 153       user_name = getpass.getuser()
 154     except Exception:  # The possible set of exceptions is not documented.
 155       user_format = ''
 156     else:
 157       user_format = '.%s' % user_name
 158
 159   tempdir = tempfile.gettempdir()
 160
 161   gcd_path = os.path.join(tempdir,
 162                           'appengine-gcd-war.%s%s' % (app_id, user_format))
 163
 164   if not os.path.exists(gcd_path):
 165     os.mkdir(gcd_path, 0700)
 166     os.mkdir(os.path.join(gcd_path, 'WEB-INF'), 0700)
 167
 168   with open(os.path.join(gcd_path, 'WEB-INF', 'web.xml'), 'w') as f:
 169     f.write("""<?xml version="1.0" encoding="UTF-8"?>
 170 <web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5">
 171
 172   <security-constraint>
 173     <web-resource-collection>
 174       <url-pattern>/datastore/*</url-pattern>
 175     </web-resource-collection>
 176     <user-data-constraint>
 177       <transport-guarantee>CONFIDENTIAL</transport-guarantee>
 178     </user-data-constraint>
 179   </security-constraint>
 180
 181   <servlet>
 182     <servlet-name>DatastoreApiServlet</servlet-name>
 183     <servlet-class>
 184       com.google.apphosting.client.datastoreservice.app.DatastoreApiServlet
 185     </servlet-class>
 186     <load-on-startup>1</load-on-startup>
 187   </servlet>
 188
 189   <servlet-mapping>
 190     <servlet-name>DatastoreApiServlet</servlet-name>
 191     <url-pattern>/datastore/*</url-pattern>
 192   </servlet-mapping>
 193
 194 </web-app>""")
 195
 196   gcd_app_xml = os.path.join(gcd_path, 'WEB-INF', 'appengine-web.xml')
 197   with open(gcd_app_xml, 'w') as f:
 198     f.write("""<?xml version="1.0" encoding="utf-8"?>
 199 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 200   <application>%s</application>
 201   <version>1</version>
 202   <module>google-cloud-datastore</module>
 203
 204   <precompilation-enabled>true</precompilation-enabled>
 205   <threadsafe>true</threadsafe>
 206
 207 </appengine-web-app>""" % app_id)
 208
 209   return gcd_app_xml
 210
 211
 212 class PortParser(object):
 213   """A parser for ints that represent ports."""
 214
 215   def __init__(self, allow_port_zero=True):
 216     self._min_port = 0 if allow_port_zero else 1
 217
 218   def __call__(self, value):
 219     try:
 220       port = int(value)
 221     except ValueError:
 222       raise argparse.ArgumentTypeError('Invalid port: %r' % value)
 223     if port < self._min_port or port >= (1 << 16):
 224       raise argparse.ArgumentTypeError('Invalid port: %d' % port)
 225     return port
 226
 227
 228 def parse_per_module_option(
 229     value, value_type, value_predicate,
 230     single_bad_type_error, single_bad_predicate_error,
 231     multiple_bad_type_error, multiple_bad_predicate_error,
 232     multiple_duplicate_module_error):
 233   """Parses command line options that may be specified per-module.
 234
 235   Args:
 236     value: A str containing the flag value to parse. Two formats are supported:
 237         1. A universal value (may not contain a colon as that is use to
 238            indicate a per-module value).
 239         2. Per-module values. One or more comma separated module-value pairs.
 240            Each pair is a module_name:value. An empty module-name is shorthand
 241            for "default" to match how not specifying a module name in the yaml
 242            is the same as specifying "module: default".
 243     value_type: a callable that converts the string representation of the value
 244         to the actual value. Should raise ValueError if the string can not
 245         be converted.
 246     value_predicate: a predicate to call on the converted value to validate
 247         the converted value. Use "lambda _: True" if all values are valid.
 248     single_bad_type_error: the message to use if a universal value is provided
 249         and value_type throws a ValueError. The message must consume a single
 250         format parameter (the provided value).
 251     single_bad_predicate_error: the message to use if a universal value is
 252         provided and value_predicate returns False. The message does not
 253         get any format parameters.
 254     multiple_bad_type_error: the message to use if a per-module value
 255         either does not have two values separated by a single colon or if
 256         value_types throws a ValueError on the second string. The message must
 257         consume a single format parameter (the module_name:value pair).
 258     multiple_bad_predicate_error: the message to use if a per-module value if
 259         value_predicate returns False. The message must consume a single format
 260         parameter (the module name).
 261     multiple_duplicate_module_error: the message to use if the same module is
 262         repeated. The message must consume a single formater parameter (the
 263         module name).
 264
 265   Returns:
 266     Either a single value of value_type for universal values or a dict of
 267     str->value_type for per-module values.
 268
 269   Raises:
 270     argparse.ArgumentTypeError: the value is invalid.
 271   """
 272   if ':' not in value:
 273     try:
 274       single_value = value_type(value)
 275     except ValueError:
 276       raise argparse.ArgumentTypeError(single_bad_type_error % value)
 277     else:
 278       if not value_predicate(single_value):
 279         raise argparse.ArgumentTypeError(single_bad_predicate_error)
 280       return single_value
 281   else:
 282     module_to_value = {}
 283     for module_value in value.split(','):
 284       try:
 285         module_name, single_value = module_value.split(':')
 286         single_value = value_type(single_value)
 287       except ValueError:
 288         raise argparse.ArgumentTypeError(multiple_bad_type_error % module_value)
 289       else:
 290         module_name = module_name.strip()
 291         if not module_name:
 292           module_name = appinfo.DEFAULT_MODULE
 293         if module_name in module_to_value:
 294           raise argparse.ArgumentTypeError(
 295               multiple_duplicate_module_error % module_name)
 296         if not value_predicate(single_value):
 297           raise argparse.ArgumentTypeError(
 298               multiple_bad_predicate_error % module_name)
 299         module_to_value[module_name] = single_value
 300     return module_to_value
 301
 302
 303 def parse_max_module_instances(value):
 304   """Returns the parsed value for the --max_module_instances flag.
 305
 306   Args:
 307     value: A str containing the flag value for parse. The format should follow
 308         one of the following examples:
 309           1. "5" - All modules are limited to 5 instances.
 310           2. "default:3,backend:20" - The default module can have 3 instances,
 311              "backend" can have 20 instances and all other modules are
 312               unaffected. An empty name (i.e. ":3") is shorthand for default
 313               to match how not specifying a module name in the yaml is the
 314               same as specifying "module: default".
 315   Returns:
 316     The parsed value of the max_module_instances flag. May either be an int
 317     (for values of the form "5") or a dict of str->int (for values of the
 318     form "default:3,backend:20").
 319
 320   Raises:
 321     argparse.ArgumentTypeError: the value is invalid.
 322   """
 323   return parse_per_module_option(
 324       value, int, lambda instances: instances > 0,
 325       'Invalid max instance count: %r',
 326       'Max instance count must be greater than zero',
 327       'Expected "module:max_instance_count": %r',
 328       'Max instance count for module %s must be greater than zero',
 329       'Duplicate max instance count for module %s')
 330
 331
 332 def parse_threadsafe_override(value):
 333   """Returns the parsed value for the --threadsafe_override flag.
 334
 335   Args:
 336     value: A str containing the flag value for parse. The format should follow
 337         one of the following examples:
 338           1. "False" - All modules override the YAML threadsafe configuration
 339              as if the YAML contained False.
 340           2. "default:False,backend:True" - The default module overrides the
 341              YAML threadsafe configuration as if the YAML contained False, the
 342              "backend" module overrides with a value of True and all other
 343              modules use the value in the YAML file. An empty name (i.e.
 344              ":True") is shorthand for default to match how not specifying a
 345              module name in the yaml is the same as specifying
 346              "module: default".
 347   Returns:
 348     The parsed value of the threadsafe_override flag. May either be a bool
 349     (for values of the form "False") or a dict of str->bool (for values of the
 350     form "default:False,backend:True").
 351
 352   Raises:
 353     argparse.ArgumentTypeError: the value is invalid.
 354   """
 355   return parse_per_module_option(
 356       value, boolean_action.BooleanParse, lambda _: True,
 357       'Invalid threadsafe override: %r',
 358       None,
 359       'Expected "module:threadsafe_override": %r',
 360       None,
 361       'Duplicate threadsafe override value for module %s')
 362
 363
 364 def parse_path(value):
 365   """Returns the given path with ~ and environment variables expanded."""
 366   return os.path.expanduser(os.path.expandvars(value))
 367
 368
 369 def create_command_line_parser():
 370   """Returns an argparse.ArgumentParser to parse command line arguments."""
 371   # TODO: Add more robust argument validation. Consider what flags
 372   # are actually needed.
 373
 374   parser = argparse.ArgumentParser(
 375       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 376   arg_name = 'yaml_path'
 377   arg_help = 'Path to a yaml file, or a directory containing yaml files'
 378   if application_configuration.java_supported():
 379     arg_name = 'yaml_or_war_path'
 380     arg_help += ', or a directory containing WEB-INF/web.xml'
 381   parser.add_argument(
 382       'config_paths', metavar=arg_name, nargs='+', help=arg_help)
 383
 384   common_group = parser.add_argument_group('Common')
 385   common_group.add_argument(
 386       '--host', default='localhost',
 387       help='host name to which application modules should bind')
 388   common_group.add_argument(
 389       '--port', type=PortParser(), default=8080,
 390       help='lowest port to which application modules should bind')
 391   common_group.add_argument(
 392       '--admin_host', default='localhost',
 393       help='host name to which the admin server should bind')
 394   common_group.add_argument(
 395       '--admin_port', type=PortParser(), default=8000,
 396       help='port to which the admin server should bind')
 397   common_group.add_argument(
 398       '--auth_domain', default='gmail.com',
 399       help='name of the authorization domain to use')
 400   common_group.add_argument(
 401       '--storage_path', metavar='PATH',
 402       type=parse_path,
 403       help='path to the data (datastore, blobstore, etc.) associated with the '
 404       'application.')
 405   common_group.add_argument(
 406       '--log_level', default='info',
 407       choices=_LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(),
 408       help='the log level below which logging messages generated by '
 409       'application code will not be displayed on the console')
 410   common_group.add_argument(
 411       '--max_module_instances',
 412       type=parse_max_module_instances,
 413       help='the maximum number of runtime instances that can be started for a '
 414       'particular module - the value can be an integer, in what case all '
 415       'modules are limited to that number of instances or a comma-seperated '
 416       'list of module:max_instances e.g. "default:5,backend:3"')
 417   common_group.add_argument(
 418       '--use_mtime_file_watcher',
 419       action=boolean_action.BooleanAction,
 420       const=True,
 421       default=False,
 422       help='use mtime polling for detecting source code changes - useful if '
 423       'modifying code from a remote machine using a distributed file system')
 424   common_group.add_argument(
 425       '--threadsafe_override',
 426       type=parse_threadsafe_override,
 427       help='override the application\'s threadsafe configuration - the value '
 428       'can be a boolean, in which case all modules threadsafe setting will '
 429       'be overridden or a comma-separated list of module:threadsafe_override '
 430       'e.g. "default:False,backend:True"')
 431   common_group.add_argument('--docker_daemon_url', help=argparse.SUPPRESS)
 432
 433   # PHP
 434   php_group = parser.add_argument_group('PHP')
 435   php_group.add_argument('--php_executable_path', metavar='PATH',
 436                          type=parse_path,
 437                          default=_get_default_php_path(),
 438                          help='path to the PHP executable')
 439   php_group.add_argument('--php_remote_debugging',
 440                          action=boolean_action.BooleanAction,
 441                          const=True,
 442                          default=False,
 443                          help='enable XDebug remote debugging')
 444
 445   # App Identity
 446   appidentity_group = parser.add_argument_group('Application Identity')
 447   appidentity_group.add_argument(
 448       '--appidentity_email_address',
 449       help='email address associated with a service account that has a '
 450       'downloadable key. May be None for no local application identity.')
 451   appidentity_group.add_argument(
 452       '--appidentity_private_key_path',
 453       help='path to private key file associated with service account '
 454       '(.pem format). Must be set if appidentity_email_address is set.')
 455
 456   # Python
 457   python_group = parser.add_argument_group('Python')
 458   python_group.add_argument(
 459       '--python_startup_script',
 460       help='the script to run at the startup of new Python runtime instances '
 461       '(useful for tools such as debuggers.')
 462   python_group.add_argument(
 463       '--python_startup_args',
 464       help='the arguments made available to the script specified in '
 465       '--python_startup_script.')
 466
 467   # Blobstore
 468   blobstore_group = parser.add_argument_group('Blobstore API')
 469   blobstore_group.add_argument(
 470       '--blobstore_path',
 471       type=parse_path,
 472       help='path to directory used to store blob contents '
 473       '(defaults to a subdirectory of --storage_path if not set)',
 474       default=None)
 475
 476   # Cloud SQL
 477   cloud_sql_group = parser.add_argument_group('Cloud SQL')
 478   cloud_sql_group.add_argument(
 479       '--mysql_host',
 480       default='localhost',
 481       help='host name of a running MySQL server used for simulated Google '
 482       'Cloud SQL storage')
 483   cloud_sql_group.add_argument(
 484       '--mysql_port', type=PortParser(allow_port_zero=False),
 485       default=3306,
 486       help='port number of a running MySQL server used for simulated Google '
 487       'Cloud SQL storage')
 488   cloud_sql_group.add_argument(
 489       '--mysql_user',
 490       default='',
 491       help='username to use when connecting to the MySQL server specified in '
 492       '--mysql_host and --mysql_port or --mysql_socket')
 493   cloud_sql_group.add_argument(
 494       '--mysql_password',
 495       default='',
 496       help='password to use when connecting to the MySQL server specified in '
 497       '--mysql_host and --mysql_port or --mysql_socket')
 498   cloud_sql_group.add_argument(
 499       '--mysql_socket',
 500       help='path to a Unix socket file to use when connecting to a running '
 501       'MySQL server used for simulated Google Cloud SQL storage')
 502
 503   # Datastore
 504   datastore_group = parser.add_argument_group('Datastore API')
 505   datastore_group.add_argument(
 506       '--datastore_path',
 507       type=parse_path,
 508       default=None,
 509       help='path to a file used to store datastore contents '
 510       '(defaults to a file in --storage_path if not set)',)
 511   datastore_group.add_argument('--clear_datastore',
 512                                action=boolean_action.BooleanAction,
 513                                const=True,
 514                                default=False,
 515                                help='clear the datastore on startup')
 516   datastore_group.add_argument(
 517       '--datastore_consistency_policy',
 518       default='time',
 519       choices=['consistent', 'random', 'time'],
 520       help='the policy to apply when deciding whether a datastore write should '
 521       'appear in global queries')
 522   datastore_group.add_argument(
 523       '--require_indexes',
 524       action=boolean_action.BooleanAction,
 525       const=True,
 526       default=False,
 527       help='generate an error on datastore queries that '
 528       'requires a composite index not found in index.yaml')
 529   datastore_group.add_argument(
 530       '--auto_id_policy',
 531       default=datastore_stub_util.SCATTERED,
 532       choices=[datastore_stub_util.SEQUENTIAL,
 533                datastore_stub_util.SCATTERED],
 534       help='the type of sequence from which the datastore stub '
 535       'assigns automatic IDs. NOTE: Sequential IDs are '
 536       'deprecated. This flag will be removed in a future '
 537       'release. Please do not rely on sequential IDs in your '
 538       'tests.')
 539   datastore_group.add_argument(
 540       '--enable_cloud_datastore',
 541       action=boolean_action.BooleanAction,
 542       const=True,
 543       default=False,
 544       help=argparse.SUPPRESS #'enable the Google Cloud Datastore API.'
 545       )
 546
 547   # Logs
 548   logs_group = parser.add_argument_group('Logs API')
 549   logs_group.add_argument(
 550       '--logs_path', default=None,
 551       help='path to a file used to store request logs (defaults to a file in '
 552       '--storage_path if not set)',)
 553
 554   # Mail
 555   mail_group = parser.add_argument_group('Mail API')
 556   mail_group.add_argument(
 557       '--show_mail_body',
 558       action=boolean_action.BooleanAction,
 559       const=True,
 560       default=False,
 561       help='logs the contents of e-mails sent using the Mail API')
 562   mail_group.add_argument(
 563       '--enable_sendmail',
 564       action=boolean_action.BooleanAction,
 565       const=True,
 566       default=False,
 567       help='use the "sendmail" tool to transmit e-mail sent '
 568       'using the Mail API (ignored if --smtp_host is set)')
 569   mail_group.add_argument(
 570       '--smtp_host', default='',
 571       help='host name of an SMTP server to use to transmit '
 572       'e-mail sent using the Mail API')
 573   mail_group.add_argument(
 574       '--smtp_port', default=25,
 575       type=PortParser(allow_port_zero=False),
 576       help='port number of an SMTP server to use to transmit '
 577       'e-mail sent using the Mail API (ignored if --smtp_host '
 578       'is not set)')
 579   mail_group.add_argument(
 580       '--smtp_user', default='',
 581       help='username to use when connecting to the SMTP server '
 582       'specified in --smtp_host and --smtp_port')
 583   mail_group.add_argument(
 584       '--smtp_password', default='',
 585       help='password to use when connecting to the SMTP server '
 586       'specified in --smtp_host and --smtp_port')
 587
 588   # Matcher
 589   prospective_search_group = parser.add_argument_group('Prospective Search API')
 590   prospective_search_group.add_argument(
 591       '--prospective_search_path', default=None,
 592       type=parse_path,
 593       help='path to a file used to store the prospective '
 594       'search subscription index (defaults to a file in '
 595       '--storage_path if not set)')
 596   prospective_search_group.add_argument(
 597       '--clear_prospective_search',
 598       action=boolean_action.BooleanAction,
 599       const=True,
 600       default=False,
 601       help='clear the prospective search subscription index')
 602
 603   # Search
 604   search_group = parser.add_argument_group('Search API')
 605   search_group.add_argument(
 606       '--search_indexes_path', default=None,
 607       type=parse_path,
 608       help='path to a file used to store search indexes '
 609       '(defaults to a file in --storage_path if not set)',)
 610   search_group.add_argument(
 611       '--clear_search_indexes',
 612       action=boolean_action.BooleanAction,
 613       const=True,
 614       default=False,
 615       help='clear the search indexes')
 616
 617   # Taskqueue
 618   taskqueue_group = parser.add_argument_group('Task Queue API')
 619   taskqueue_group.add_argument(
 620       '--enable_task_running',
 621       action=boolean_action.BooleanAction,
 622       const=True,
 623       default=True,
 624       help='run "push" tasks created using the taskqueue API automatically')
 625
 626   # Misc
 627   misc_group = parser.add_argument_group('Miscellaneous')
 628   misc_group.add_argument(
 629       '--allow_skipped_files',
 630       action=boolean_action.BooleanAction,
 631       const=True,
 632       default=False,
 633       help='make files specified in the app.yaml "skip_files" or "static" '
 634       'handles readable by the application.')
 635   # No help to avoid lengthening help message for rarely used feature:
 636   # host name to which the server for API calls should bind.
 637   misc_group.add_argument(
 638       '--api_host', default='localhost',
 639       help=argparse.SUPPRESS)
 640   misc_group.add_argument(
 641       '--api_port', type=PortParser(), default=0,
 642       help='port to which the server for API calls should bind')
 643   misc_group.add_argument(
 644       '--automatic_restart',
 645       action=boolean_action.BooleanAction,
 646       const=True,
 647       default=True,
 648       help=('restart instances automatically when files relevant to their '
 649             'module are changed'))
 650   misc_group.add_argument(
 651       '--dev_appserver_log_level', default='info',
 652       choices=_LOG_LEVEL_TO_PYTHON_CONSTANT.keys(),
 653       help='the log level below which logging messages generated by '
 654       'the development server will not be displayed on the console (this '
 655       'flag is more useful for diagnosing problems in dev_appserver.py rather '
 656       'than in application code)')
 657   misc_group.add_argument(
 658       '--skip_sdk_update_check',
 659       action=boolean_action.BooleanAction,
 660       const=True,
 661       default=False,
 662       help='skip checking for SDK updates (if false, use .appcfg_nag to '
 663       'decide)')
 664   misc_group.add_argument(
 665       '--default_gcs_bucket_name', default=None,
 666       help='default Google Cloud Storgage bucket name')
 667
 668
 669   return parser
 670
 671 PARSER = create_command_line_parser()
 672
 673
 674 def _clear_datastore_storage(datastore_path):
 675   """Delete the datastore storage file at the given path."""
 676   # lexists() returns True for broken symlinks, where exists() returns False.
 677   if os.path.lexists(datastore_path):
 678     try:
 679       os.remove(datastore_path)
 680     except OSError, e:
 681       logging.warning('Failed to remove datastore file %r: %s',
 682                       datastore_path,
 683                       e)
 684
 685
 686 def _clear_prospective_search_storage(prospective_search_path):
 687   """Delete the perspective search storage file at the given path."""
 688   # lexists() returns True for broken symlinks, where exists() returns False.
 689   if os.path.lexists(prospective_search_path):
 690     try:
 691       os.remove(prospective_search_path)
 692     except OSError, e:
 693       logging.warning('Failed to remove prospective search file %r: %s',
 694                       prospective_search_path,
 695                       e)
 696
 697
 698 def _clear_search_indexes_storage(search_index_path):
 699   """Delete the search indexes storage file at the given path."""
 700   # lexists() returns True for broken symlinks, where exists() returns False.
 701   if os.path.lexists(search_index_path):
 702     try:
 703       os.remove(search_index_path)
 704     except OSError, e:
 705       logging.warning('Failed to remove search indexes file %r: %s',
 706                       search_index_path,
 707                       e)
 708
 709
 710 def _setup_environ(app_id):
 711   """Sets up the os.environ dictionary for the front-end server and API server.
 712
 713   This function should only be called once.
 714
 715   Args:
 716     app_id: The id of the application.
 717   """
 718   os.environ['APPLICATION_ID'] = app_id
 719
 720
 721 class DevelopmentServer(object):
 722   """Encapsulates the logic for the development server.
 723
 724   Only a single instance of the class may be created per process. See
 725   _setup_environ.
 726   """
 727
 728   def __init__(self):
 729     # A list of servers that are currently running.
 730     self._running_modules = []
 731     self._module_to_port = {}
 732     self._dispatcher = None
 733
 734   def module_to_address(self, module_name, instance=None):
 735     """Returns the address of a module."""
 736
 737     if module_name is None:
 738       return self._dispatcher.dispatch_address
 739     return self._dispatcher.get_hostname(
 740         module_name,
 741         self._dispatcher.get_default_version(module_name),
 742         instance)
 743
 744   def start(self, options):
 745     """Start devappserver2 servers based on the provided command line arguments.
 746
 747     Args:
 748       options: An argparse.Namespace containing the command line arguments.
 749     """
 750     logging.getLogger().setLevel(
 751         _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
 752
 753     configuration = application_configuration.ApplicationConfiguration(
 754         options.config_paths)
 755
 756     if options.enable_cloud_datastore:
 757       # This requires the oauth server stub to return that the logged in user
 758       # is in fact an admin.
 759       os.environ['OAUTH_IS_ADMIN'] = '1'
 760       gcd_module = application_configuration.ModuleConfiguration(
 761           _generate_gcd_app(configuration.app_id.split('~')[1]))
 762       configuration.modules.append(gcd_module)
 763
 764     if options.skip_sdk_update_check:
 765       logging.info('Skipping SDK update check.')
 766     else:
 767       update_checker.check_for_updates(configuration)
 768
 769     # There is no good way to set the default encoding from application code
 770     # (it needs to be done during interpreter initialization in site.py or
 771     # sitecustomize.py) so just warn developers if they have a different
 772     # encoding than production.
 773     if sys.getdefaultencoding() != _PROD_DEFAULT_ENCODING:
 774       logging.warning(
 775           'The default encoding of your local Python interpreter is set to %r '
 776           'while App Engine\'s production environment uses %r; as a result '
 777           'your code may behave differently when deployed.',
 778           sys.getdefaultencoding(), _PROD_DEFAULT_ENCODING)
 779
 780     if options.port == 0:
 781       logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
 782                    '--port=0')
 783
 784     _setup_environ(configuration.app_id)
 785
 786     self._dispatcher = dispatcher.Dispatcher(
 787         configuration,
 788         options.host,
 789         options.port,
 790         options.auth_domain,
 791         _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
 792         self._create_php_config(options),
 793         self._create_python_config(options),
 794         self._create_cloud_sql_config(options),
 795         self._create_vm_config(options),
 796         self._create_module_to_setting(options.max_module_instances,
 797                                        configuration, '--max_module_instances'),
 798         options.use_mtime_file_watcher,
 799         options.automatic_restart,
 800         options.allow_skipped_files,
 801         self._create_module_to_setting(options.threadsafe_override,
 802                                        configuration, '--threadsafe_override'))
 803
 804     request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
 805     storage_path = _get_storage_path(options.storage_path, configuration.app_id)
 806
 807     apis = self._create_api_server(
 808         request_data, storage_path, options, configuration)
 809     apis.start()
 810     self._running_modules.append(apis)
 811
 812     self._dispatcher.start(options.api_host, apis.port, request_data)
 813
 814     xsrf_path = os.path.join(storage_path, 'xsrf')
 815     admin = admin_server.AdminServer(options.admin_host, options.admin_port,
 816                                      self._dispatcher, configuration, xsrf_path)
 817     admin.start()
 818     self._running_modules.append(admin)
 819
 820   def stop(self):
 821     """Stops all running devappserver2 modules."""
 822     while self._running_modules:
 823       self._running_modules.pop().quit()
 824     if self._dispatcher:
 825       self._dispatcher.quit()
 826
 827   @staticmethod
 828   def _create_api_server(request_data, storage_path, options, configuration):
 829     datastore_path = options.datastore_path or os.path.join(storage_path,
 830                                                             'datastore.db')
 831     logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
 832
 833     search_index_path = options.search_indexes_path or os.path.join(
 834         storage_path, 'search_indexes')
 835
 836     prospective_search_path = options.prospective_search_path or os.path.join(
 837         storage_path, 'prospective-search')
 838
 839     blobstore_path = options.blobstore_path or os.path.join(storage_path,
 840                                                             'blobs')
 841
 842     if options.clear_datastore:
 843       _clear_datastore_storage(datastore_path)
 844
 845     if options.clear_prospective_search:
 846       _clear_prospective_search_storage(prospective_search_path)
 847
 848     if options.clear_search_indexes:
 849       _clear_search_indexes_storage(search_index_path)
 850
 851     if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
 852       logging.warn("--auto_id_policy='sequential' is deprecated. This option "
 853                    "will be removed in a future release.")
 854
 855     application_address = '%s' % options.host
 856     if options.port and options.port != 80:
 857       application_address += ':' + str(options.port)
 858
 859     user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
 860                                      login.CONTINUE_PARAM)
 861     user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
 862                                     login.LOGOUT_ACTION)
 863
 864     if options.datastore_consistency_policy == 'time':
 865       consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
 866     elif options.datastore_consistency_policy == 'random':
 867       consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
 868     elif options.datastore_consistency_policy == 'consistent':
 869       consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
 870     else:
 871       assert 0, ('unknown consistency policy: %r' %
 872                  options.datastore_consistency_policy)
 873
 874     api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
 875         configuration.app_id, datastore_path)
 876     api_server.setup_stubs(
 877         request_data=request_data,
 878         app_id=configuration.app_id,
 879         application_root=configuration.modules[0].application_root,
 880         # The "trusted" flag is only relevant for Google administrative
 881         # applications.
 882         trusted=getattr(options, 'trusted', False),
 883         appidentity_email_address=options.appidentity_email_address,
 884         appidentity_private_key_path=os.path.abspath(
 885             options.appidentity_private_key_path)
 886         if options.appidentity_private_key_path else None,
 887         blobstore_path=blobstore_path,
 888         datastore_path=datastore_path,
 889         datastore_consistency=consistency,
 890         datastore_require_indexes=options.require_indexes,
 891         datastore_auto_id_policy=options.auto_id_policy,
 892         images_host_prefix='http://%s' % application_address,
 893         logs_path=logs_path,
 894         mail_smtp_host=options.smtp_host,
 895         mail_smtp_port=options.smtp_port,
 896         mail_smtp_user=options.smtp_user,
 897         mail_smtp_password=options.smtp_password,
 898         mail_enable_sendmail=options.enable_sendmail,
 899         mail_show_mail_body=options.show_mail_body,
 900         matcher_prospective_search_path=prospective_search_path,
 901         search_index_path=search_index_path,
 902         taskqueue_auto_run_tasks=options.enable_task_running,
 903         taskqueue_default_http_server=application_address,
 904         user_login_url=user_login_url,
 905         user_logout_url=user_logout_url,
 906         default_gcs_bucket_name=options.default_gcs_bucket_name)
 907
 908     return api_server.APIServer(options.api_host, options.api_port,
 909                                 configuration.app_id)
 910
 911   @staticmethod
 912   def _create_php_config(options):
 913     php_config = runtime_config_pb2.PhpConfig()
 914     if options.php_executable_path:
 915       php_config.php_executable_path = os.path.abspath(
 916           options.php_executable_path)
 917     php_config.enable_debugger = options.php_remote_debugging
 918     return php_config
 919
 920   @staticmethod
 921   def _create_python_config(options):
 922     python_config = runtime_config_pb2.PythonConfig()
 923     if options.python_startup_script:
 924       python_config.startup_script = os.path.abspath(
 925           options.python_startup_script)
 926       if options.python_startup_args:
 927         python_config.startup_args = options.python_startup_args
 928     return python_config
 929
 930   @staticmethod
 931   def _create_cloud_sql_config(options):
 932     cloud_sql_config = runtime_config_pb2.CloudSQL()
 933     cloud_sql_config.mysql_host = options.mysql_host
 934     cloud_sql_config.mysql_port = options.mysql_port
 935     cloud_sql_config.mysql_user = options.mysql_user
 936     cloud_sql_config.mysql_password = options.mysql_password
 937     if options.mysql_socket:
 938       cloud_sql_config.mysql_socket = options.mysql_socket
 939     return cloud_sql_config
 940
 941   @staticmethod
 942   def _create_vm_config(options):
 943     vm_config = runtime_config_pb2.VMConfig()
 944     if options.docker_daemon_url:
 945       vm_config.docker_daemon_url = options.docker_daemon_url
 946     return vm_config
 947
 948   @staticmethod
 949   def _create_module_to_setting(setting, configuration, option):
 950     """Create a per-module dictionary configuration.
 951
 952     Creates a dictionary that maps a module name to a configuration
 953     setting. Used in conjunction with parse_per_module_option.
 954
 955     Args:
 956       setting: a value that can be None, a dict of str->type or a single value.
 957       configuration: an ApplicationConfiguration object.
 958       option: the option name the setting came from.
 959
 960     Returns:
 961       A dict of str->type.
 962     """
 963     if setting is None:
 964       return {}
 965
 966     module_names = [module_configuration.module_name
 967                     for module_configuration in configuration.modules]
 968     if isinstance(setting, dict):
 969       # Warn and remove a setting if the module name is unknown.
 970       module_to_setting = {}
 971       for module_name, value in setting.items():
 972         if module_name in module_names:
 973           module_to_setting[module_name] = value
 974         else:
 975           logging.warning('Unknown module %r for %r', module_name, option)
 976       return module_to_setting
 977
 978     # Create a dict with an entry for every module.
 979     return {module_name: setting for module_name in module_names}
 980
 981
 982 def main():
 983   shutdown.install_signal_handlers()
 984   # The timezone must be set in the devappserver2 process rather than just in
 985   # the runtime so printed log timestamps are consistent and the taskqueue stub
 986   # expects the timezone to be UTC. The runtime inherits the environment.
 987   os.environ['TZ'] = 'UTC'
 988   if hasattr(time, 'tzset'):
 989     # time.tzet() should be called on Unix, but doesn't exist on Windows.
 990     time.tzset()
 991   options = PARSER.parse_args()
 992   dev_server = DevelopmentServer()
 993   try:
 994     dev_server.start(options)
 995     shutdown.wait_until_shutdown()
 996   finally:
 997     dev_server.stop()
 998
 999
1000 if __name__ == '__main__':
1001   main()