python/google/appengine/tools/devappserver2/application_configuration.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Stores application configuration taken from e.g. app.yaml, queues.yaml."""
  18
  19 # TODO: Support more than just app.yaml.
  20
  21
  22 import errno
  23 import logging
  24 import os
  25 import os.path
  26 import random
  27 import string
  28 import threading
  29 import types
  30
  31 from google.appengine.api import appinfo
  32 from google.appengine.api import appinfo_includes
  33 from google.appengine.api import backendinfo
  34 from google.appengine.api import dispatchinfo
  35 from google.appengine.tools.devappserver2 import errors
  36
  37 # Constants passed to functions registered with
  38 # ServerConfiguration.add_change_callback.
  39 NORMALIZED_LIBRARIES_CHANGED = 1
  40 SKIP_FILES_CHANGED = 2
  41 HANDLERS_CHANGED = 3
  42 INBOUND_SERVICES_CHANGED = 4
  43 ENV_VARIABLES_CHANGED = 5
  44 ERROR_HANDLERS_CHANGED = 6
  45 NOBUILD_FILES_CHANGED = 7
  46
  47
  48 class ServerConfiguration(object):
  49   """Stores server configuration information.
  50
  51   Most configuration options are mutable and may change any time
  52   check_for_updates is called. Client code must be able to cope with these
  53   changes.
  54
  55   Other properties are immutable (see _IMMUTABLE_PROPERTIES) and are guaranteed
  56   to be constant for the lifetime of the instance.
  57   """
  58
  59   _IMMUTABLE_PROPERTIES = [
  60       ('application', 'application'),
  61       ('version', 'major_version'),
  62       ('runtime', 'runtime'),
  63       ('threadsafe', 'threadsafe'),
  64       ('server', 'server_name'),
  65       ('basic_scaling', 'basic_scaling'),
  66       ('manual_scaling', 'manual_scaling'),
  67       ('automatic_scaling', 'automatic_scaling')]
  68
  69   def __init__(self, yaml_path):
  70     """Initializer for ServerConfiguration.
  71
  72     Args:
  73       yaml_path: A string containing the full path of the yaml file containing
  74           the configuration for this server.
  75     """
  76     self._yaml_path = yaml_path
  77     self._app_info_external = None
  78     self._application_root = os.path.realpath(os.path.dirname(yaml_path))
  79     self._last_failure_message = None
  80
  81     self._app_info_external, files_to_check = self._parse_configuration(
  82         self._yaml_path)
  83     self._mtimes = self._get_mtimes([self._yaml_path] + files_to_check)
  84     self._application = 'dev~%s' % self._app_info_external.application
  85     self._api_version = self._app_info_external.api_version
  86     self._server_name = self._app_info_external.server
  87     self._version = self._app_info_external.version
  88     self._threadsafe = self._app_info_external.threadsafe
  89     self._basic_scaling = self._app_info_external.basic_scaling
  90     self._manual_scaling = self._app_info_external.manual_scaling
  91     self._automatic_scaling = self._app_info_external.automatic_scaling
  92     self._runtime = self._app_info_external.runtime
  93     if self._runtime == 'python':
  94       logging.warning(
  95           'The "python" runtime specified in "%s" is not supported - the '
  96           '"python27" runtime will be used instead. A description of the '
  97           'differences between the two can be found here:\n'
  98           'https://developers.google.com/appengine/docs/python/python25/diff27',
  99            self._yaml_path)
 100     self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 101                                      range(18))
 102
 103   @property
 104   def application_root(self):
 105     """The directory containing the application e.g. "/home/user/myapp"."""
 106     return self._application_root
 107
 108   @property
 109   def application(self):
 110     return self._application
 111
 112   @property
 113   def api_version(self):
 114     return self._api_version
 115
 116   @property
 117   def server_name(self):
 118     return self._server_name or 'default'
 119
 120   @property
 121   def major_version(self):
 122     return self._version
 123
 124   @property
 125   def version_id(self):
 126     if self.server_name == 'default':
 127       return '%s.%s' % (
 128           self.major_version,
 129           self._minor_version_id)
 130     else:
 131       return '%s:%s.%s' % (
 132           self.server_name,
 133           self.major_version,
 134           self._minor_version_id)
 135
 136   @property
 137   def runtime(self):
 138     return self._runtime
 139
 140   @property
 141   def threadsafe(self):
 142     return self._threadsafe
 143
 144   @property
 145   def basic_scaling(self):
 146     return self._basic_scaling
 147
 148   @property
 149   def manual_scaling(self):
 150     return self._manual_scaling
 151
 152   @property
 153   def automatic_scaling(self):
 154     return self._automatic_scaling
 155
 156   @property
 157   def normalized_libraries(self):
 158     return self._app_info_external.GetNormalizedLibraries()
 159
 160   @property
 161   def skip_files(self):
 162     return self._app_info_external.skip_files
 163
 164   @property
 165   def nobuild_files(self):
 166     return self._app_info_external.nobuild_files
 167
 168   @property
 169   def error_handlers(self):
 170     return self._app_info_external.error_handlers
 171
 172   @property
 173   def handlers(self):
 174     return self._app_info_external.handlers
 175
 176   @property
 177   def inbound_services(self):
 178     return self._app_info_external.inbound_services
 179
 180   @property
 181   def env_variables(self):
 182     return self._app_info_external.env_variables
 183
 184   @property
 185   def is_backend(self):
 186     return False
 187
 188   def check_for_updates(self):
 189     """Return any configuration changes since the last check_for_updates call.
 190
 191     Returns:
 192       A set containing the changes that occured. See the *_CHANGED module
 193       constants.
 194     """
 195     new_mtimes = self._get_mtimes(self._mtimes.keys())
 196     if new_mtimes == self._mtimes:
 197       return set()
 198
 199     try:
 200       app_info_external, files_to_check = self._parse_configuration(
 201           self._yaml_path)
 202     except Exception, e:
 203       failure_message = str(e)
 204       if failure_message != self._last_failure_message:
 205         logging.error('Configuration is not valid: %s', failure_message)
 206       self._last_failure_message = failure_message
 207       return set()
 208     self._last_failure_message = None
 209
 210     self._mtimes = self._get_mtimes([self._yaml_path] + files_to_check)
 211
 212     for app_info_attribute, self_attribute in self._IMMUTABLE_PROPERTIES:
 213       app_info_value = getattr(app_info_external, app_info_attribute)
 214       self_value = getattr(self, self_attribute)
 215       if (app_info_value == self_value or
 216           app_info_value == getattr(self._app_info_external,
 217                                     app_info_attribute)):
 218         # Only generate a warning if the value is both different from the
 219         # immutable value *and* different from the last loaded value.
 220         continue
 221
 222       if isinstance(app_info_value, types.StringTypes):
 223         logging.warning('Restart the development server to see updates to "%s" '
 224                         '["%s" => "%s"]',
 225                         app_info_attribute,
 226                         self_value,
 227                         app_info_value)
 228       else:
 229         logging.warning('Restart the development server to see updates to "%s"',
 230                         app_info_attribute)
 231
 232     changes = set()
 233     if (app_info_external.GetNormalizedLibraries() !=
 234         self.normalized_libraries):
 235       changes.add(NORMALIZED_LIBRARIES_CHANGED)
 236     if app_info_external.skip_files != self.skip_files:
 237       changes.add(SKIP_FILES_CHANGED)
 238     if app_info_external.nobuild_files != self.nobuild_files:
 239       changes.add(NOBUILD_FILES_CHANGED)
 240     if app_info_external.handlers != self.handlers:
 241       changes.add(HANDLERS_CHANGED)
 242     if app_info_external.inbound_services != self.inbound_services:
 243       changes.add(INBOUND_SERVICES_CHANGED)
 244     if app_info_external.env_variables != self.env_variables:
 245       changes.add(ENV_VARIABLES_CHANGED)
 246     if app_info_external.error_handlers != self.error_handlers:
 247       changes.add(ERROR_HANDLERS_CHANGED)
 248
 249     self._app_info_external = app_info_external
 250     if changes:
 251       self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 252                                        range(18))
 253     return changes
 254
 255   @staticmethod
 256   def _get_mtimes(filenames):
 257     filename_to_mtime = {}
 258     for filename in filenames:
 259       try:
 260         filename_to_mtime[filename] = os.path.getmtime(filename)
 261       except OSError as e:
 262         # Ignore deleted includes.
 263         if e.errno != errno.ENOENT:
 264           raise
 265     return filename_to_mtime
 266
 267   @staticmethod
 268   def _parse_configuration(configuration_path):
 269     # TODO: It probably makes sense to catch the exception raised
 270     # by Parse() and re-raise it using a module-specific exception.
 271     with open(configuration_path) as f:
 272       return appinfo_includes.ParseAndReturnIncludePaths(f)
 273
 274
 275 class BackendsConfiguration(object):
 276   """Stores configuration information for a backends.yaml file."""
 277
 278   def __init__(self, app_yaml_path, backend_yaml_path):
 279     """Initializer for BackendsConfiguration.
 280
 281     Args:
 282       app_yaml_path: A string containing the full path of the yaml file
 283           containing the configuration for this server.
 284       backend_yaml_path: A string containing the full path of the backends.yaml
 285           file containing the configuration for backends.
 286     """
 287     self._update_lock = threading.RLock()
 288     self._base_server_configuration = ServerConfiguration(app_yaml_path)
 289     backend_info_external = self._parse_configuration(
 290         backend_yaml_path)
 291
 292     self._backends_name_to_backend_entry = {}
 293     for backend in backend_info_external.backends or []:
 294       self._backends_name_to_backend_entry[backend.name] = backend
 295     self._changes = dict(
 296         (backend_name, set())
 297         for backend_name in self._backends_name_to_backend_entry)
 298
 299   @staticmethod
 300   def _parse_configuration(configuration_path):
 301     # TODO: It probably makes sense to catch the exception raised
 302     # by Parse() and re-raise it using a module-specific exception.
 303     with open(configuration_path) as f:
 304       return backendinfo.LoadBackendInfo(f)
 305
 306   def get_backend_configurations(self):
 307     return [BackendConfiguration(self._base_server_configuration, self, entry)
 308             for entry in self._backends_name_to_backend_entry.values()]
 309
 310   def check_for_updates(self, backend_name):
 311     """Return any configuration changes since the last check_for_updates call.
 312
 313     Args:
 314       backend_name: A str containing the name of the backend to be checked for
 315           updates.
 316
 317     Returns:
 318       A set containing the changes that occured. See the *_CHANGED module
 319       constants.
 320     """
 321     with self._update_lock:
 322       server_changes = self._base_server_configuration.check_for_updates()
 323       if server_changes:
 324         for backend_changes in self._changes.values():
 325           backend_changes.update(server_changes)
 326       changes = self._changes[backend_name]
 327       self._changes[backend_name] = set()
 328     return changes
 329
 330
 331 class BackendConfiguration(object):
 332   """Stores backend configuration information.
 333
 334   This interface is and must remain identical to ServerConfiguration.
 335   """
 336
 337   def __init__(self, server_configuration, backends_configuration,
 338                backend_entry):
 339     """Initializer for BackendConfiguration.
 340
 341     Args:
 342       server_configuration: A ServerConfiguration to use.
 343       backends_configuration: The BackendsConfiguration that tracks updates for
 344           this BackendConfiguration.
 345       backend_entry: A backendinfo.BackendEntry containing the backend
 346           configuration.
 347     """
 348     self._server_configuration = server_configuration
 349     self._backends_configuration = backends_configuration
 350     self._backend_entry = backend_entry
 351
 352     if backend_entry.dynamic:
 353       self._basic_scaling = appinfo.BasicScaling(
 354           max_instances=backend_entry.instances or 1)
 355       self._manual_scaling = None
 356     else:
 357       self._basic_scaling = None
 358       self._manual_scaling = appinfo.ManualScaling(
 359           instances=backend_entry.instances or 1)
 360     self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 361                                      range(18))
 362
 363   @property
 364   def application_root(self):
 365     """The directory containing the application e.g. "/home/user/myapp"."""
 366     return self._server_configuration.application_root
 367
 368   @property
 369   def application(self):
 370     return self._server_configuration.application
 371
 372   @property
 373   def api_version(self):
 374     return self._server_configuration.api_version
 375
 376   @property
 377   def server_name(self):
 378     return self._backend_entry.name
 379
 380   @property
 381   def major_version(self):
 382     return self._server_configuration.major_version
 383
 384   @property
 385   def version_id(self):
 386     return '%s:%s.%s' % (
 387         self.server_name,
 388         self.major_version,
 389         self._minor_version_id)
 390
 391   @property
 392   def runtime(self):
 393     return self._server_configuration.runtime
 394
 395   @property
 396   def threadsafe(self):
 397     return self._server_configuration.threadsafe
 398
 399   @property
 400   def basic_scaling(self):
 401     return self._basic_scaling
 402
 403   @property
 404   def manual_scaling(self):
 405     return self._manual_scaling
 406
 407   @property
 408   def automatic_scaling(self):
 409     return None
 410
 411   @property
 412   def normalized_libraries(self):
 413     return self._server_configuration.normalized_libraries
 414
 415   @property
 416   def skip_files(self):
 417     return self._server_configuration.skip_files
 418
 419   @property
 420   def nobuild_files(self):
 421     return self._server_configuration.nobuild_files
 422
 423   @property
 424   def error_handlers(self):
 425     return self._server_configuration.error_handlers
 426
 427   @property
 428   def handlers(self):
 429     if self._backend_entry.start:
 430       return [appinfo.URLMap(
 431           url='/_ah/start',
 432           script=self._backend_entry.start,
 433           login='admin')] + self._server_configuration.handlers
 434     return self._server_configuration.handlers
 435
 436   @property
 437   def inbound_services(self):
 438     return self._server_configuration.inbound_services
 439
 440   @property
 441   def env_variables(self):
 442     return self._server_configuration.env_variables
 443
 444   @property
 445   def is_backend(self):
 446     return True
 447
 448   def check_for_updates(self):
 449     """Return any configuration changes since the last check_for_updates call.
 450
 451     Returns:
 452       A set containing the changes that occured. See the *_CHANGED module
 453       constants.
 454     """
 455     changes = self._backends_configuration.check_for_updates(
 456         self._backend_entry.name)
 457     if changes:
 458       self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 459                                        range(18))
 460     return changes
 461
 462
 463 class DispatchConfiguration(object):
 464   """Stores dispatcher configuration information."""
 465
 466   def __init__(self, yaml_path):
 467     self._yaml_path = yaml_path
 468     self._mtime = os.path.getmtime(self._yaml_path)
 469     self._process_dispatch_entries(self._parse_configuration(self._yaml_path))
 470
 471   @staticmethod
 472   def _parse_configuration(configuration_path):
 473     # TODO: It probably makes sense to catch the exception raised
 474     # by LoadSingleDispatch() and re-raise it using a module-specific exception.
 475     with open(configuration_path) as f:
 476       return dispatchinfo.LoadSingleDispatch(f)
 477
 478   def check_for_updates(self):
 479     mtime = os.path.getmtime(self._yaml_path)
 480     if mtime > self._mtime:
 481       self._mtime = mtime
 482       try:
 483         dispatch_info_external = self._parse_configuration(self._yaml_path)
 484       except Exception, e:
 485         failure_message = str(e)
 486         logging.error('Configuration is not valid: %s', failure_message)
 487         return
 488       self._process_dispatch_entries(dispatch_info_external)
 489
 490   def _process_dispatch_entries(self, dispatch_info_external):
 491     path_only_entries = []
 492     hostname_entries = []
 493     for entry in dispatch_info_external.dispatch:
 494       parsed_url = dispatchinfo.ParsedURL(entry.url)
 495       if parsed_url.host:
 496         hostname_entries.append(entry)
 497       else:
 498         path_only_entries.append((parsed_url, entry.server))
 499     if hostname_entries:
 500       logging.warning(
 501           'Hostname routing is not supported by the development server. The '
 502           'following dispatch entries will not match any requests:\n%s',
 503           '\n\t'.join(str(entry) for entry in hostname_entries))
 504     self._entries = path_only_entries
 505
 506   @property
 507   def dispatch(self):
 508     return self._entries
 509
 510
 511 class ApplicationConfiguration(object):
 512   """Stores application configuration information."""
 513
 514   def __init__(self, yaml_paths):
 515     """Initializer for ApplicationConfiguration.
 516
 517     Args:
 518       yaml_paths: A list of strings containing the paths to yaml files.
 519     """
 520     self.servers = []
 521     self.dispatch = None
 522     if len(yaml_paths) == 1 and os.path.isdir(yaml_paths[0]):
 523       directory_path = yaml_paths[0]
 524       for app_yaml_path in [os.path.join(directory_path, 'app.yaml'),
 525                             os.path.join(directory_path, 'app.yml')]:
 526         if os.path.exists(app_yaml_path):
 527           yaml_paths = [app_yaml_path]
 528           break
 529       else:
 530         raise errors.AppConfigNotFoundError(
 531             'no app.yaml file at %r' % directory_path)
 532       for backends_yaml_path in [os.path.join(directory_path, 'backends.yaml'),
 533                                  os.path.join(directory_path, 'backends.yml')]:
 534         if os.path.exists(backends_yaml_path):
 535           yaml_paths.append(backends_yaml_path)
 536           break
 537     for yaml_path in yaml_paths:
 538       if os.path.isdir(yaml_path):
 539         raise errors.InvalidAppConfigError(
 540             '"%s" is a directory and a yaml configuration file is required' %
 541             yaml_path)
 542       elif (yaml_path.endswith('backends.yaml') or
 543             yaml_path.endswith('backends.yml')):
 544         # TODO: Reuse the ServerConfiguration created for the app.yaml
 545         # instead of creating another one for the same file.
 546         self.servers.extend(
 547             BackendsConfiguration(yaml_path.replace('backends.y', 'app.y'),
 548                                   yaml_path).get_backend_configurations())
 549       elif (yaml_path.endswith('dispatch.yaml') or
 550             yaml_path.endswith('dispatch.yml')):
 551         if self.dispatch:
 552           raise errors.InvalidAppConfigError(
 553               'Multiple dispatch.yaml files specified')
 554         self.dispatch = DispatchConfiguration(yaml_path)
 555       else:
 556         server_configuration = ServerConfiguration(yaml_path)
 557         self.servers.append(server_configuration)
 558     application_ids = set(server.application
 559                           for server in self.servers)
 560     if len(application_ids) > 1:
 561       raise errors.InvalidAppConfigError(
 562           'More than one application ID found: %s' %
 563           ', '.join(sorted(application_ids)))
 564
 565     self._app_id = application_ids.pop()
 566     server_names = set()
 567     for server in self.servers:
 568       if server.server_name in server_names:
 569         raise errors.InvalidAppConfigError('Duplicate server: %s' %
 570                                            server.server_name)
 571       server_names.add(server.server_name)
 572     if self.dispatch:
 573       if 'default' not in server_names:
 574         raise errors.InvalidAppConfigError(
 575             'A default server must be specified.')
 576       missing_servers = (
 577           set(server_name for _, server_name in self.dispatch.dispatch) -
 578           server_names)
 579       if missing_servers:
 580         raise errors.InvalidAppConfigError(
 581             'Servers %s specified in dispatch.yaml are not defined by a yaml '
 582             'file.' % sorted(missing_servers))
 583
 584   @property
 585   def app_id(self):
 586     return self._app_id