python/google/appengine/tools/devappserver2/devappserver2.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """The main entry point for the new development server."""
  18
  19
  20 import argparse
  21 import errno
  22 import getpass
  23 import itertools
  24 import logging
  25 import os
  26 import sys
  27 import tempfile
  28 import time
  29
  30 from google.appengine.api import appinfo
  31 from google.appengine.datastore import datastore_stub_util
  32 from google.appengine.tools import boolean_action
  33 from google.appengine.tools.devappserver2 import api_server
  34 from google.appengine.tools.devappserver2 import application_configuration
  35 from google.appengine.tools.devappserver2 import dispatcher
  36 from google.appengine.tools.devappserver2 import gcd_application
  37 from google.appengine.tools.devappserver2 import login
  38 from google.appengine.tools.devappserver2 import runtime_config_pb2
  39 from google.appengine.tools.devappserver2 import shutdown
  40 from google.appengine.tools.devappserver2 import update_checker
  41 from google.appengine.tools.devappserver2 import wsgi_request_info
  42 from google.appengine.tools.devappserver2.admin import admin_server
  43
  44 # Initialize logging early -- otherwise some library packages may
  45 # pre-empt our log formatting.  NOTE: the level is provisional; it may
  46 # be changed in main() based on the --debug flag.
  47 logging.basicConfig(
  48     level=logging.INFO,
  49     format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
  50
  51 # Valid choices for --log_level and their corresponding constants in
  52 # runtime_config_pb2.Config.stderr_log_level.
  53 _LOG_LEVEL_TO_RUNTIME_CONSTANT = {
  54     'debug': 0,
  55     'info': 1,
  56     'warning': 2,
  57     'error': 3,
  58     'critical': 4,
  59 }
  60
  61 # Valid choices for --dev_appserver_log_level and their corresponding Python
  62 # logging levels
  63 _LOG_LEVEL_TO_PYTHON_CONSTANT = {
  64     'debug': logging.DEBUG,
  65     'info': logging.INFO,
  66     'warning': logging.WARNING,
  67     'error': logging.ERROR,
  68     'critical': logging.CRITICAL,
  69 }
  70
  71 # The default encoding used by the production interpreter.
  72 _PROD_DEFAULT_ENCODING = 'ascii'
  73
  74
  75 def _generate_storage_paths(app_id):
  76   """Yield an infinite sequence of possible storage paths."""
  77   if sys.platform == 'win32':
  78     # The temp directory is per-user on Windows so there is no reason to add
  79     # the username to the generated directory name.
  80     user_format = ''
  81   else:
  82     try:
  83       user_name = getpass.getuser()
  84     except Exception:  # The possible set of exceptions is not documented.
  85       user_format = ''
  86     else:
  87       user_format = '.%s' % user_name
  88
  89   tempdir = tempfile.gettempdir()
  90   yield os.path.join(tempdir, 'appengine.%s%s' % (app_id, user_format))
  91   for i in itertools.count(1):
  92     yield os.path.join(tempdir, 'appengine.%s%s.%d' % (app_id, user_format, i))
  93
  94
  95 def _get_storage_path(path, app_id):
  96   """Returns a path to the directory where stub data can be stored."""
  97   _, _, app_id = app_id.replace(':', '_').rpartition('~')
  98   if path is None:
  99     for path in _generate_storage_paths(app_id):
 100       try:
 101         os.mkdir(path, 0700)
 102       except OSError, e:
 103         if e.errno == errno.EEXIST:
 104           # Check that the directory is only accessable by the current user to
 105           # protect against an attacker creating the directory in advance in
 106           # order to access any created files. Windows has per-user temporary
 107           # directories and st_mode does not include per-user permission
 108           # information so assume that it is safe.
 109           if sys.platform == 'win32' or (
 110               (os.stat(path).st_mode & 0777) == 0700 and os.path.isdir(path)):
 111             return path
 112           else:
 113             continue
 114         raise
 115       else:
 116         return path
 117   elif not os.path.exists(path):
 118     os.mkdir(path)
 119     return path
 120   elif not os.path.isdir(path):
 121     raise IOError('the given storage path %r is a file, a directory was '
 122                   'expected' % path)
 123   else:
 124     return path
 125
 126
 127 def _get_default_php_path():
 128   """Returns the path to the siloed php-cgi binary or None if not present."""
 129   default_php_executable_path = None
 130   if sys.platform == 'win32':
 131     default_php_executable_path = os.path.abspath(
 132         os.path.join(os.path.dirname(sys.argv[0]),
 133                      'php/php-5.4-Win32-VC9-x86/php-cgi.exe'))
 134   elif sys.platform == 'darwin':
 135     # The Cloud SDK uses symlinks in its packaging of the Mac Launcher.  First
 136     # try to find PHP relative to the apsolute path of this executable.  If that
 137     # doesn't work, try using the path without dereferencing all symlinks.
 138     base_paths = [os.path.realpath(sys.argv[0]), sys.argv[0]]
 139     for base_path in base_paths:
 140       default_php_executable_path = os.path.abspath(
 141           os.path.join(os.path.dirname(os.path.dirname(base_path)), 'php-cgi'))
 142       if os.path.exists(default_php_executable_path):
 143         break
 144
 145   if (default_php_executable_path and
 146       os.path.exists(default_php_executable_path)):
 147     return default_php_executable_path
 148   return None
 149
 150
 151 class PortParser(object):
 152   """A parser for ints that represent ports."""
 153
 154   def __init__(self, allow_port_zero=True):
 155     self._min_port = 0 if allow_port_zero else 1
 156
 157   def __call__(self, value):
 158     try:
 159       port = int(value)
 160     except ValueError:
 161       raise argparse.ArgumentTypeError('Invalid port: %r' % value)
 162     if port < self._min_port or port >= (1 << 16):
 163       raise argparse.ArgumentTypeError('Invalid port: %d' % port)
 164     return port
 165
 166
 167 def parse_per_module_option(
 168     value, value_type, value_predicate,
 169     single_bad_type_error, single_bad_predicate_error,
 170     multiple_bad_type_error, multiple_bad_predicate_error,
 171     multiple_duplicate_module_error):
 172   """Parses command line options that may be specified per-module.
 173
 174   Args:
 175     value: A str containing the flag value to parse. Two formats are supported:
 176         1. A universal value (may not contain a colon as that is use to
 177            indicate a per-module value).
 178         2. Per-module values. One or more comma separated module-value pairs.
 179            Each pair is a module_name:value. An empty module-name is shorthand
 180            for "default" to match how not specifying a module name in the yaml
 181            is the same as specifying "module: default".
 182     value_type: a callable that converts the string representation of the value
 183         to the actual value. Should raise ValueError if the string can not
 184         be converted.
 185     value_predicate: a predicate to call on the converted value to validate
 186         the converted value. Use "lambda _: True" if all values are valid.
 187     single_bad_type_error: the message to use if a universal value is provided
 188         and value_type throws a ValueError. The message must consume a single
 189         format parameter (the provided value).
 190     single_bad_predicate_error: the message to use if a universal value is
 191         provided and value_predicate returns False. The message does not
 192         get any format parameters.
 193     multiple_bad_type_error: the message to use if a per-module value
 194         either does not have two values separated by a single colon or if
 195         value_types throws a ValueError on the second string. The message must
 196         consume a single format parameter (the module_name:value pair).
 197     multiple_bad_predicate_error: the message to use if a per-module value if
 198         value_predicate returns False. The message must consume a single format
 199         parameter (the module name).
 200     multiple_duplicate_module_error: the message to use if the same module is
 201         repeated. The message must consume a single formater parameter (the
 202         module name).
 203
 204   Returns:
 205     Either a single value of value_type for universal values or a dict of
 206     str->value_type for per-module values.
 207
 208   Raises:
 209     argparse.ArgumentTypeError: the value is invalid.
 210   """
 211   if ':' not in value:
 212     try:
 213       single_value = value_type(value)
 214     except ValueError:
 215       raise argparse.ArgumentTypeError(single_bad_type_error % value)
 216     else:
 217       if not value_predicate(single_value):
 218         raise argparse.ArgumentTypeError(single_bad_predicate_error)
 219       return single_value
 220   else:
 221     module_to_value = {}
 222     for module_value in value.split(','):
 223       try:
 224         module_name, single_value = module_value.split(':')
 225         single_value = value_type(single_value)
 226       except ValueError:
 227         raise argparse.ArgumentTypeError(multiple_bad_type_error % module_value)
 228       else:
 229         module_name = module_name.strip()
 230         if not module_name:
 231           module_name = appinfo.DEFAULT_MODULE
 232         if module_name in module_to_value:
 233           raise argparse.ArgumentTypeError(
 234               multiple_duplicate_module_error % module_name)
 235         if not value_predicate(single_value):
 236           raise argparse.ArgumentTypeError(
 237               multiple_bad_predicate_error % module_name)
 238         module_to_value[module_name] = single_value
 239     return module_to_value
 240
 241
 242 def parse_max_module_instances(value):
 243   """Returns the parsed value for the --max_module_instances flag.
 244
 245   Args:
 246     value: A str containing the flag value for parse. The format should follow
 247         one of the following examples:
 248           1. "5" - All modules are limited to 5 instances.
 249           2. "default:3,backend:20" - The default module can have 3 instances,
 250              "backend" can have 20 instances and all other modules are
 251               unaffected. An empty name (i.e. ":3") is shorthand for default
 252               to match how not specifying a module name in the yaml is the
 253               same as specifying "module: default".
 254   Returns:
 255     The parsed value of the max_module_instances flag. May either be an int
 256     (for values of the form "5") or a dict of str->int (for values of the
 257     form "default:3,backend:20").
 258
 259   Raises:
 260     argparse.ArgumentTypeError: the value is invalid.
 261   """
 262   return parse_per_module_option(
 263       value, int, lambda instances: instances > 0,
 264       'Invalid max instance count: %r',
 265       'Max instance count must be greater than zero',
 266       'Expected "module:max_instance_count": %r',
 267       'Max instance count for module %s must be greater than zero',
 268       'Duplicate max instance count for module %s')
 269
 270
 271 def parse_threadsafe_override(value):
 272   """Returns the parsed value for the --threadsafe_override flag.
 273
 274   Args:
 275     value: A str containing the flag value for parse. The format should follow
 276         one of the following examples:
 277           1. "False" - All modules override the YAML threadsafe configuration
 278              as if the YAML contained False.
 279           2. "default:False,backend:True" - The default module overrides the
 280              YAML threadsafe configuration as if the YAML contained False, the
 281              "backend" module overrides with a value of True and all other
 282              modules use the value in the YAML file. An empty name (i.e.
 283              ":True") is shorthand for default to match how not specifying a
 284              module name in the yaml is the same as specifying
 285              "module: default".
 286   Returns:
 287     The parsed value of the threadsafe_override flag. May either be a bool
 288     (for values of the form "False") or a dict of str->bool (for values of the
 289     form "default:False,backend:True").
 290
 291   Raises:
 292     argparse.ArgumentTypeError: the value is invalid.
 293   """
 294   return parse_per_module_option(
 295       value, boolean_action.BooleanParse, lambda _: True,
 296       'Invalid threadsafe override: %r',
 297       None,
 298       'Expected "module:threadsafe_override": %r',
 299       None,
 300       'Duplicate threadsafe override value for module %s')
 301
 302
 303 def parse_path(value):
 304   """Returns the given path with ~ and environment variables expanded."""
 305   return os.path.expanduser(os.path.expandvars(value))
 306
 307
 308 def create_command_line_parser():
 309   """Returns an argparse.ArgumentParser to parse command line arguments."""
 310   # TODO: Add more robust argument validation. Consider what flags
 311   # are actually needed.
 312
 313   parser = argparse.ArgumentParser(
 314       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 315   arg_name = 'yaml_path'
 316   arg_help = 'Path to a yaml file, or a directory containing yaml files'
 317   if application_configuration.java_supported():
 318     arg_name = 'yaml_or_war_path'
 319     arg_help += ', or a directory containing WEB-INF/web.xml'
 320   parser.add_argument(
 321       'config_paths', metavar=arg_name, nargs='+', help=arg_help)
 322
 323   common_group = parser.add_argument_group('Common')
 324   common_group.add_argument(
 325       '-A', '--application', action='store', dest='app_id',
 326       help='Set the application, overriding the application value from the '
 327       'app.yaml file.')
 328   common_group.add_argument(
 329       '--host', default='localhost',
 330       help='host name to which application modules should bind')
 331   common_group.add_argument(
 332       '--port', type=PortParser(), default=8080,
 333       help='lowest port to which application modules should bind')
 334   common_group.add_argument(
 335       '--admin_host', default='localhost',
 336       help='host name to which the admin server should bind')
 337   common_group.add_argument(
 338       '--admin_port', type=PortParser(), default=8000,
 339       help='port to which the admin server should bind')
 340   common_group.add_argument(
 341       '--auth_domain', default='gmail.com',
 342       help='name of the authorization domain to use')
 343   common_group.add_argument(
 344       '--storage_path', metavar='PATH',
 345       type=parse_path,
 346       help='path to the data (datastore, blobstore, etc.) associated with the '
 347       'application.')
 348   common_group.add_argument(
 349       '--log_level', default='info',
 350       choices=_LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(),
 351       help='the log level below which logging messages generated by '
 352       'application code will not be displayed on the console')
 353   common_group.add_argument(
 354       '--max_module_instances',
 355       type=parse_max_module_instances,
 356       help='the maximum number of runtime instances that can be started for a '
 357       'particular module - the value can be an integer, in what case all '
 358       'modules are limited to that number of instances or a comma-seperated '
 359       'list of module:max_instances e.g. "default:5,backend:3"')
 360   common_group.add_argument(
 361       '--use_mtime_file_watcher',
 362       action=boolean_action.BooleanAction,
 363       const=True,
 364       default=False,
 365       help='use mtime polling for detecting source code changes - useful if '
 366       'modifying code from a remote machine using a distributed file system')
 367   common_group.add_argument(
 368       '--threadsafe_override',
 369       type=parse_threadsafe_override,
 370       help='override the application\'s threadsafe configuration - the value '
 371       'can be a boolean, in which case all modules threadsafe setting will '
 372       'be overridden or a comma-separated list of module:threadsafe_override '
 373       'e.g. "default:False,backend:True"')
 374   common_group.add_argument('--docker_daemon_url', help=argparse.SUPPRESS)
 375
 376   # PHP
 377   php_group = parser.add_argument_group('PHP')
 378   php_group.add_argument('--php_executable_path', metavar='PATH',
 379                          type=parse_path,
 380                          default=_get_default_php_path(),
 381                          help='path to the PHP executable')
 382   php_group.add_argument('--php_remote_debugging',
 383                          action=boolean_action.BooleanAction,
 384                          const=True,
 385                          default=False,
 386                          help='enable XDebug remote debugging')
 387
 388   # Dart
 389   dart_group = parser.add_argument_group('Dart')
 390   dart_group.add_argument('--dart_sdk', help=argparse.SUPPRESS)
 391   dart_group.add_argument('--dart_dev_mode',
 392                           choices=['dev', 'deploy'],
 393                           help=argparse.SUPPRESS)
 394   dart_group.add_argument('--dart_pub_serve_host', help=argparse.SUPPRESS)
 395   dart_group.add_argument('--dart_pub_serve_port',
 396                           type=PortParser(), help=argparse.SUPPRESS)
 397
 398   # App Identity
 399   appidentity_group = parser.add_argument_group('Application Identity')
 400   appidentity_group.add_argument(
 401       '--appidentity_email_address',
 402       help='email address associated with a service account that has a '
 403       'downloadable key. May be None for no local application identity.')
 404   appidentity_group.add_argument(
 405       '--appidentity_private_key_path',
 406       help='path to private key file associated with service account '
 407       '(.pem format). Must be set if appidentity_email_address is set.')
 408
 409   # Python
 410   python_group = parser.add_argument_group('Python')
 411   python_group.add_argument(
 412       '--python_startup_script',
 413       help='the script to run at the startup of new Python runtime instances '
 414       '(useful for tools such as debuggers.')
 415   python_group.add_argument(
 416       '--python_startup_args',
 417       help='the arguments made available to the script specified in '
 418       '--python_startup_script.')
 419
 420   # Blobstore
 421   blobstore_group = parser.add_argument_group('Blobstore API')
 422   blobstore_group.add_argument(
 423       '--blobstore_path',
 424       type=parse_path,
 425       help='path to directory used to store blob contents '
 426       '(defaults to a subdirectory of --storage_path if not set)',
 427       default=None)
 428
 429   # Cloud SQL
 430   cloud_sql_group = parser.add_argument_group('Cloud SQL')
 431   cloud_sql_group.add_argument(
 432       '--mysql_host',
 433       default='localhost',
 434       help='host name of a running MySQL server used for simulated Google '
 435       'Cloud SQL storage')
 436   cloud_sql_group.add_argument(
 437       '--mysql_port', type=PortParser(allow_port_zero=False),
 438       default=3306,
 439       help='port number of a running MySQL server used for simulated Google '
 440       'Cloud SQL storage')
 441   cloud_sql_group.add_argument(
 442       '--mysql_user',
 443       default='',
 444       help='username to use when connecting to the MySQL server specified in '
 445       '--mysql_host and --mysql_port or --mysql_socket')
 446   cloud_sql_group.add_argument(
 447       '--mysql_password',
 448       default='',
 449       help='password to use when connecting to the MySQL server specified in '
 450       '--mysql_host and --mysql_port or --mysql_socket')
 451   cloud_sql_group.add_argument(
 452       '--mysql_socket',
 453       help='path to a Unix socket file to use when connecting to a running '
 454       'MySQL server used for simulated Google Cloud SQL storage')
 455
 456   # Datastore
 457   datastore_group = parser.add_argument_group('Datastore API')
 458   datastore_group.add_argument(
 459       '--datastore_path',
 460       type=parse_path,
 461       default=None,
 462       help='path to a file used to store datastore contents '
 463       '(defaults to a file in --storage_path if not set)',)
 464   datastore_group.add_argument('--clear_datastore',
 465                                action=boolean_action.BooleanAction,
 466                                const=True,
 467                                default=False,
 468                                help='clear the datastore on startup')
 469   datastore_group.add_argument(
 470       '--datastore_consistency_policy',
 471       default='time',
 472       choices=['consistent', 'random', 'time'],
 473       help='the policy to apply when deciding whether a datastore write should '
 474       'appear in global queries')
 475   datastore_group.add_argument(
 476       '--require_indexes',
 477       action=boolean_action.BooleanAction,
 478       const=True,
 479       default=False,
 480       help='generate an error on datastore queries that '
 481       'requires a composite index not found in index.yaml')
 482   datastore_group.add_argument(
 483       '--auto_id_policy',
 484       default=datastore_stub_util.SCATTERED,
 485       choices=[datastore_stub_util.SEQUENTIAL,
 486                datastore_stub_util.SCATTERED],
 487       help='the type of sequence from which the datastore stub '
 488       'assigns automatic IDs. NOTE: Sequential IDs are '
 489       'deprecated. This flag will be removed in a future '
 490       'release. Please do not rely on sequential IDs in your '
 491       'tests.')
 492   datastore_group.add_argument(
 493       '--enable_cloud_datastore',
 494       action=boolean_action.BooleanAction,
 495       const=True,
 496       default=False,
 497       help=argparse.SUPPRESS #'enable the Google Cloud Datastore API.'
 498       )
 499
 500   # Logs
 501   logs_group = parser.add_argument_group('Logs API')
 502   logs_group.add_argument(
 503       '--logs_path', default=None,
 504       help='path to a file used to store request logs (defaults to a file in '
 505       '--storage_path if not set)',)
 506
 507   # Mail
 508   mail_group = parser.add_argument_group('Mail API')
 509   mail_group.add_argument(
 510       '--show_mail_body',
 511       action=boolean_action.BooleanAction,
 512       const=True,
 513       default=False,
 514       help='logs the contents of e-mails sent using the Mail API')
 515   mail_group.add_argument(
 516       '--enable_sendmail',
 517       action=boolean_action.BooleanAction,
 518       const=True,
 519       default=False,
 520       help='use the "sendmail" tool to transmit e-mail sent '
 521       'using the Mail API (ignored if --smtp_host is set)')
 522   mail_group.add_argument(
 523       '--smtp_host', default='',
 524       help='host name of an SMTP server to use to transmit '
 525       'e-mail sent using the Mail API')
 526   mail_group.add_argument(
 527       '--smtp_port', default=25,
 528       type=PortParser(allow_port_zero=False),
 529       help='port number of an SMTP server to use to transmit '
 530       'e-mail sent using the Mail API (ignored if --smtp_host '
 531       'is not set)')
 532   mail_group.add_argument(
 533       '--smtp_user', default='',
 534       help='username to use when connecting to the SMTP server '
 535       'specified in --smtp_host and --smtp_port')
 536   mail_group.add_argument(
 537       '--smtp_password', default='',
 538       help='password to use when connecting to the SMTP server '
 539       'specified in --smtp_host and --smtp_port')
 540   mail_group.add_argument(
 541       '--smtp_allow_tls',
 542       action=boolean_action.BooleanAction,
 543       const=True,
 544       default=True,
 545       help='Allow TLS to be used when the SMTP server announces TLS support '
 546       '(ignored if --smtp_host is not set)')
 547
 548   # Matcher
 549   prospective_search_group = parser.add_argument_group('Prospective Search API')
 550   prospective_search_group.add_argument(
 551       '--prospective_search_path', default=None,
 552       type=parse_path,
 553       help='path to a file used to store the prospective '
 554       'search subscription index (defaults to a file in '
 555       '--storage_path if not set)')
 556   prospective_search_group.add_argument(
 557       '--clear_prospective_search',
 558       action=boolean_action.BooleanAction,
 559       const=True,
 560       default=False,
 561       help='clear the prospective search subscription index')
 562
 563   # Search
 564   search_group = parser.add_argument_group('Search API')
 565   search_group.add_argument(
 566       '--search_indexes_path', default=None,
 567       type=parse_path,
 568       help='path to a file used to store search indexes '
 569       '(defaults to a file in --storage_path if not set)',)
 570   search_group.add_argument(
 571       '--clear_search_indexes',
 572       action=boolean_action.BooleanAction,
 573       const=True,
 574       default=False,
 575       help='clear the search indexes')
 576
 577   # Taskqueue
 578   taskqueue_group = parser.add_argument_group('Task Queue API')
 579   taskqueue_group.add_argument(
 580       '--enable_task_running',
 581       action=boolean_action.BooleanAction,
 582       const=True,
 583       default=True,
 584       help='run "push" tasks created using the taskqueue API automatically')
 585
 586   # Misc
 587   misc_group = parser.add_argument_group('Miscellaneous')
 588   misc_group.add_argument(
 589       '--allow_skipped_files',
 590       action=boolean_action.BooleanAction,
 591       const=True,
 592       default=False,
 593       help='make files specified in the app.yaml "skip_files" or "static" '
 594       'handles readable by the application.')
 595   # No help to avoid lengthening help message for rarely used feature:
 596   # host name to which the server for API calls should bind.
 597   misc_group.add_argument(
 598       '--api_host', default='localhost',
 599       help=argparse.SUPPRESS)
 600   misc_group.add_argument(
 601       '--api_port', type=PortParser(), default=0,
 602       help='port to which the server for API calls should bind')
 603   misc_group.add_argument(
 604       '--automatic_restart',
 605       action=boolean_action.BooleanAction,
 606       const=True,
 607       default=True,
 608       help=('restart instances automatically when files relevant to their '
 609             'module are changed'))
 610   misc_group.add_argument(
 611       '--dev_appserver_log_level', default='info',
 612       choices=_LOG_LEVEL_TO_PYTHON_CONSTANT.keys(),
 613       help='the log level below which logging messages generated by '
 614       'the development server will not be displayed on the console (this '
 615       'flag is more useful for diagnosing problems in dev_appserver.py rather '
 616       'than in application code)')
 617   misc_group.add_argument(
 618       '--skip_sdk_update_check',
 619       action=boolean_action.BooleanAction,
 620       const=True,
 621       default=False,
 622       help='skip checking for SDK updates (if false, use .appcfg_nag to '
 623       'decide)')
 624   misc_group.add_argument(
 625       '--default_gcs_bucket_name', default=None,
 626       help='default Google Cloud Storgage bucket name')
 627
 628
 629   return parser
 630
 631 PARSER = create_command_line_parser()
 632
 633
 634 def _clear_datastore_storage(datastore_path):
 635   """Delete the datastore storage file at the given path."""
 636   # lexists() returns True for broken symlinks, where exists() returns False.
 637   if os.path.lexists(datastore_path):
 638     try:
 639       os.remove(datastore_path)
 640     except OSError, e:
 641       logging.warning('Failed to remove datastore file %r: %s',
 642                       datastore_path,
 643                       e)
 644
 645
 646 def _clear_prospective_search_storage(prospective_search_path):
 647   """Delete the perspective search storage file at the given path."""
 648   # lexists() returns True for broken symlinks, where exists() returns False.
 649   if os.path.lexists(prospective_search_path):
 650     try:
 651       os.remove(prospective_search_path)
 652     except OSError, e:
 653       logging.warning('Failed to remove prospective search file %r: %s',
 654                       prospective_search_path,
 655                       e)
 656
 657
 658 def _clear_search_indexes_storage(search_index_path):
 659   """Delete the search indexes storage file at the given path."""
 660   # lexists() returns True for broken symlinks, where exists() returns False.
 661   if os.path.lexists(search_index_path):
 662     try:
 663       os.remove(search_index_path)
 664     except OSError, e:
 665       logging.warning('Failed to remove search indexes file %r: %s',
 666                       search_index_path,
 667                       e)
 668
 669
 670 def _setup_environ(app_id):
 671   """Sets up the os.environ dictionary for the front-end server and API server.
 672
 673   This function should only be called once.
 674
 675   Args:
 676     app_id: The id of the application.
 677   """
 678   os.environ['APPLICATION_ID'] = app_id
 679
 680
 681 class DevelopmentServer(object):
 682   """Encapsulates the logic for the development server.
 683
 684   Only a single instance of the class may be created per process. See
 685   _setup_environ.
 686   """
 687
 688   def __init__(self):
 689     # A list of servers that are currently running.
 690     self._running_modules = []
 691     self._module_to_port = {}
 692     self._dispatcher = None
 693
 694   def module_to_address(self, module_name, instance=None):
 695     """Returns the address of a module."""
 696
 697     if module_name is None:
 698       return self._dispatcher.dispatch_address
 699     return self._dispatcher.get_hostname(
 700         module_name,
 701         self._dispatcher.get_default_version(module_name),
 702         instance)
 703
 704   def start(self, options):
 705     """Start devappserver2 servers based on the provided command line arguments.
 706
 707     Args:
 708       options: An argparse.Namespace containing the command line arguments.
 709     """
 710     logging.getLogger().setLevel(
 711         _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
 712
 713     configuration = application_configuration.ApplicationConfiguration(
 714         options.config_paths, options.app_id)
 715
 716     if options.enable_cloud_datastore:
 717       # This requires the oauth server stub to return that the logged in user
 718       # is in fact an admin.
 719       os.environ['OAUTH_IS_ADMIN'] = '1'
 720       gcd_module = application_configuration.ModuleConfiguration(
 721           gcd_application.generate_gcd_app(configuration.app_id.split('~')[1]))
 722       configuration.modules.append(gcd_module)
 723
 724     if options.skip_sdk_update_check:
 725       logging.info('Skipping SDK update check.')
 726     else:
 727       update_checker.check_for_updates(configuration)
 728
 729     # There is no good way to set the default encoding from application code
 730     # (it needs to be done during interpreter initialization in site.py or
 731     # sitecustomize.py) so just warn developers if they have a different
 732     # encoding than production.
 733     if sys.getdefaultencoding() != _PROD_DEFAULT_ENCODING:
 734       logging.warning(
 735           'The default encoding of your local Python interpreter is set to %r '
 736           'while App Engine\'s production environment uses %r; as a result '
 737           'your code may behave differently when deployed.',
 738           sys.getdefaultencoding(), _PROD_DEFAULT_ENCODING)
 739
 740     if options.port == 0:
 741       logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
 742                    '--port=0')
 743
 744     _setup_environ(configuration.app_id)
 745
 746     self._dispatcher = dispatcher.Dispatcher(
 747         configuration,
 748         options.host,
 749         options.port,
 750         options.auth_domain,
 751         _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
 752         self._create_php_config(options),
 753         self._create_python_config(options),
 754         self._create_cloud_sql_config(options),
 755         self._create_vm_config(options),
 756         self._create_module_to_setting(options.max_module_instances,
 757                                        configuration, '--max_module_instances'),
 758         options.use_mtime_file_watcher,
 759         options.automatic_restart,
 760         options.allow_skipped_files,
 761         self._create_module_to_setting(options.threadsafe_override,
 762                                        configuration, '--threadsafe_override'))
 763
 764     request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
 765     storage_path = _get_storage_path(options.storage_path, configuration.app_id)
 766
 767     apis = self._create_api_server(
 768         request_data, storage_path, options, configuration)
 769     apis.start()
 770     self._running_modules.append(apis)
 771
 772     self._dispatcher.start(options.api_host, apis.port, request_data)
 773
 774     xsrf_path = os.path.join(storage_path, 'xsrf')
 775     admin = admin_server.AdminServer(options.admin_host, options.admin_port,
 776                                      self._dispatcher, configuration, xsrf_path)
 777     admin.start()
 778     self._running_modules.append(admin)
 779
 780   def stop(self):
 781     """Stops all running devappserver2 modules."""
 782     while self._running_modules:
 783       self._running_modules.pop().quit()
 784     if self._dispatcher:
 785       self._dispatcher.quit()
 786
 787   @staticmethod
 788   def _create_api_server(request_data, storage_path, options, configuration):
 789     datastore_path = options.datastore_path or os.path.join(storage_path,
 790                                                             'datastore.db')
 791     logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
 792
 793     search_index_path = options.search_indexes_path or os.path.join(
 794         storage_path, 'search_indexes')
 795
 796     prospective_search_path = options.prospective_search_path or os.path.join(
 797         storage_path, 'prospective-search')
 798
 799     blobstore_path = options.blobstore_path or os.path.join(storage_path,
 800                                                             'blobs')
 801
 802     if options.clear_datastore:
 803       _clear_datastore_storage(datastore_path)
 804
 805     if options.clear_prospective_search:
 806       _clear_prospective_search_storage(prospective_search_path)
 807
 808     if options.clear_search_indexes:
 809       _clear_search_indexes_storage(search_index_path)
 810
 811     if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
 812       logging.warn("--auto_id_policy='sequential' is deprecated. This option "
 813                    "will be removed in a future release.")
 814
 815     application_address = '%s' % options.host
 816     if options.port and options.port != 80:
 817       application_address += ':' + str(options.port)
 818
 819     user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
 820                                      login.CONTINUE_PARAM)
 821     user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
 822                                     login.LOGOUT_ACTION)
 823
 824     if options.datastore_consistency_policy == 'time':
 825       consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
 826     elif options.datastore_consistency_policy == 'random':
 827       consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
 828     elif options.datastore_consistency_policy == 'consistent':
 829       consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
 830     else:
 831       assert 0, ('unknown consistency policy: %r' %
 832                  options.datastore_consistency_policy)
 833
 834     api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
 835         configuration.app_id, datastore_path)
 836     api_server.setup_stubs(
 837         request_data=request_data,
 838         app_id=configuration.app_id,
 839         application_root=configuration.modules[0].application_root,
 840         # The "trusted" flag is only relevant for Google administrative
 841         # applications.
 842         trusted=getattr(options, 'trusted', False),
 843         appidentity_email_address=options.appidentity_email_address,
 844         appidentity_private_key_path=os.path.abspath(
 845             options.appidentity_private_key_path)
 846         if options.appidentity_private_key_path else None,
 847         blobstore_path=blobstore_path,
 848         datastore_path=datastore_path,
 849         datastore_consistency=consistency,
 850         datastore_require_indexes=options.require_indexes,
 851         datastore_auto_id_policy=options.auto_id_policy,
 852         images_host_prefix='http://%s' % application_address,
 853         logs_path=logs_path,
 854         mail_smtp_host=options.smtp_host,
 855         mail_smtp_port=options.smtp_port,
 856         mail_smtp_user=options.smtp_user,
 857         mail_smtp_password=options.smtp_password,
 858         mail_enable_sendmail=options.enable_sendmail,
 859         mail_show_mail_body=options.show_mail_body,
 860         mail_allow_tls=options.smtp_allow_tls,
 861         matcher_prospective_search_path=prospective_search_path,
 862         search_index_path=search_index_path,
 863         taskqueue_auto_run_tasks=options.enable_task_running,
 864         taskqueue_default_http_server=application_address,
 865         user_login_url=user_login_url,
 866         user_logout_url=user_logout_url,
 867         default_gcs_bucket_name=options.default_gcs_bucket_name)
 868
 869     return api_server.APIServer(options.api_host, options.api_port,
 870                                 configuration.app_id)
 871
 872   @staticmethod
 873   def _create_php_config(options):
 874     php_config = runtime_config_pb2.PhpConfig()
 875     if options.php_executable_path:
 876       php_config.php_executable_path = os.path.abspath(
 877           options.php_executable_path)
 878     php_config.enable_debugger = options.php_remote_debugging
 879     return php_config
 880
 881   @staticmethod
 882   def _create_python_config(options):
 883     python_config = runtime_config_pb2.PythonConfig()
 884     if options.python_startup_script:
 885       python_config.startup_script = os.path.abspath(
 886           options.python_startup_script)
 887       if options.python_startup_args:
 888         python_config.startup_args = options.python_startup_args
 889     return python_config
 890
 891   @staticmethod
 892   def _create_cloud_sql_config(options):
 893     cloud_sql_config = runtime_config_pb2.CloudSQL()
 894     cloud_sql_config.mysql_host = options.mysql_host
 895     cloud_sql_config.mysql_port = options.mysql_port
 896     cloud_sql_config.mysql_user = options.mysql_user
 897     cloud_sql_config.mysql_password = options.mysql_password
 898     if options.mysql_socket:
 899       cloud_sql_config.mysql_socket = options.mysql_socket
 900     return cloud_sql_config
 901
 902   @staticmethod
 903   def _create_vm_config(options):
 904     vm_config = runtime_config_pb2.VMConfig()
 905     if options.docker_daemon_url:
 906       vm_config.docker_daemon_url = options.docker_daemon_url
 907     if options.dart_sdk:
 908       vm_config.dart_config.dart_sdk = os.path.abspath(options.dart_sdk)
 909     if options.dart_dev_mode:
 910       vm_config.dart_config.dart_dev_mode = options.dart_dev_mode
 911     if options.dart_pub_serve_host:
 912       vm_config.dart_config.dart_pub_serve_host = options.dart_pub_serve_host
 913     if options.dart_pub_serve_port:
 914       vm_config.dart_config.dart_pub_serve_port = options.dart_pub_serve_port
 915     return vm_config
 916
 917   @staticmethod
 918   def _create_module_to_setting(setting, configuration, option):
 919     """Create a per-module dictionary configuration.
 920
 921     Creates a dictionary that maps a module name to a configuration
 922     setting. Used in conjunction with parse_per_module_option.
 923
 924     Args:
 925       setting: a value that can be None, a dict of str->type or a single value.
 926       configuration: an ApplicationConfiguration object.
 927       option: the option name the setting came from.
 928
 929     Returns:
 930       A dict of str->type.
 931     """
 932     if setting is None:
 933       return {}
 934
 935     module_names = [module_configuration.module_name
 936                     for module_configuration in configuration.modules]
 937     if isinstance(setting, dict):
 938       # Warn and remove a setting if the module name is unknown.
 939       module_to_setting = {}
 940       for module_name, value in setting.items():
 941         if module_name in module_names:
 942           module_to_setting[module_name] = value
 943         else:
 944           logging.warning('Unknown module %r for %r', module_name, option)
 945       return module_to_setting
 946
 947     # Create a dict with an entry for every module.
 948     return {module_name: setting for module_name in module_names}
 949
 950
 951 def main():
 952   shutdown.install_signal_handlers()
 953   # The timezone must be set in the devappserver2 process rather than just in
 954   # the runtime so printed log timestamps are consistent and the taskqueue stub
 955   # expects the timezone to be UTC. The runtime inherits the environment.
 956   os.environ['TZ'] = 'UTC'
 957   if hasattr(time, 'tzset'):
 958     # time.tzet() should be called on Unix, but doesn't exist on Windows.
 959     time.tzset()
 960   options = PARSER.parse_args()
 961   dev_server = DevelopmentServer()
 962   try:
 963     dev_server.start(options)
 964     shutdown.wait_until_shutdown()
 965   finally:
 966     dev_server.stop()
 967
 968
 969 if __name__ == '__main__':
 970   main()