python/google/appengine/tools/devappserver2/application_configuration.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Stores application configuration taken from e.g. app.yaml, index.yaml."""
  18
  19 # TODO: Support more than just app.yaml.
  20
  21
  22 import errno
  23 import logging
  24 import os
  25 import os.path
  26 import random
  27 import string
  28 import threading
  29 import types
  30
  31 from google.appengine.api import appinfo
  32 from google.appengine.api import appinfo_includes
  33 from google.appengine.api import backendinfo
  34 from google.appengine.api import dispatchinfo
  35 from google.appengine.client.services import port_manager
  36 from google.appengine.tools import queue_xml_parser
  37 from google.appengine.tools import yaml_translator
  38 from google.appengine.tools.devappserver2 import errors
  39
  40 # Constants passed to functions registered with
  41 # ModuleConfiguration.add_change_callback.
  42 NORMALIZED_LIBRARIES_CHANGED = 1
  43 SKIP_FILES_CHANGED = 2
  44 HANDLERS_CHANGED = 3
  45 INBOUND_SERVICES_CHANGED = 4
  46 ENV_VARIABLES_CHANGED = 5
  47 ERROR_HANDLERS_CHANGED = 6
  48 NOBUILD_FILES_CHANGED = 7
  49
  50
  51 _HEALTH_CHECK_DEFAULTS = {
  52     'enable_health_check': True,
  53     'check_interval_sec': 5,
  54     'timeout_sec': 4,
  55     'unhealthy_threshold': 2,
  56     'healthy_threshold': 2,
  57     'restart_threshold': 60,
  58     'host': '127.0.0.1'
  59 }
  60
  61
  62 def java_supported():
  63   """True if this SDK supports running Java apps in the dev appserver."""
  64   java_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'java')
  65   return os.path.isdir(java_dir)
  66
  67
  68 class ModuleConfiguration(object):
  69   """Stores module configuration information.
  70
  71   Most configuration options are mutable and may change any time
  72   check_for_updates is called. Client code must be able to cope with these
  73   changes.
  74
  75   Other properties are immutable (see _IMMUTABLE_PROPERTIES) and are guaranteed
  76   to be constant for the lifetime of the instance.
  77   """
  78
  79   _IMMUTABLE_PROPERTIES = [
  80       ('application', 'application'),
  81       ('version', 'major_version'),
  82       ('runtime', 'runtime'),
  83       ('threadsafe', 'threadsafe'),
  84       ('module', 'module_name'),
  85       ('basic_scaling', 'basic_scaling'),
  86       ('manual_scaling', 'manual_scaling'),
  87       ('automatic_scaling', 'automatic_scaling')]
  88
  89   def __init__(self, config_path, app_id=None):
  90     """Initializer for ModuleConfiguration.
  91
  92     Args:
  93       config_path: A string containing the full path of the yaml or xml file
  94           containing the configuration for this module.
  95       app_id: A string that is the application id, or None if the application id
  96           from the yaml or xml file should be used.
  97     """
  98     self._config_path = config_path
  99     self._forced_app_id = app_id
 100     root = os.path.dirname(config_path)
 101     self._is_java = os.path.normpath(config_path).endswith(
 102         os.sep + 'WEB-INF' + os.sep + 'appengine-web.xml')
 103     if self._is_java:
 104       # We assume Java's XML-based config files only if config_path is
 105       # something like /foo/bar/WEB-INF/appengine-web.xml. In this case,
 106       # the application root is /foo/bar. Other apps, configured with YAML,
 107       # have something like /foo/bar/app.yaml, with application root /foo/bar.
 108       root = os.path.dirname(root)
 109     self._application_root = os.path.realpath(root)
 110     self._last_failure_message = None
 111
 112     self._app_info_external, files_to_check = self._parse_configuration(
 113         self._config_path)
 114     self._mtimes = self._get_mtimes(files_to_check)
 115     self._application = '%s~%s' % (self.partition,
 116                                    self.application_external_name)
 117     self._api_version = self._app_info_external.api_version
 118     self._module_name = self._app_info_external.module
 119     self._version = self._app_info_external.version
 120     self._threadsafe = self._app_info_external.threadsafe
 121     self._basic_scaling = self._app_info_external.basic_scaling
 122     self._manual_scaling = self._app_info_external.manual_scaling
 123     self._automatic_scaling = self._app_info_external.automatic_scaling
 124     self._runtime = self._app_info_external.runtime
 125     if self._runtime == 'python':
 126       logging.warning(
 127           'The "python" runtime specified in "%s" is not supported - the '
 128           '"python27" runtime will be used instead. A description of the '
 129           'differences between the two can be found here:\n'
 130           'https://developers.google.com/appengine/docs/python/python25/diff27',
 131           self._config_path)
 132     self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 133                                      range(18))
 134
 135     self._forwarded_ports = {}
 136     if self.runtime == 'vm':
 137       vm_settings = self._app_info_external.vm_settings
 138       if vm_settings:
 139         ports = vm_settings.get('forwarded_ports')
 140         if ports:
 141           logging.debug('setting forwarded ports %s', ports)
 142           pm = port_manager.PortManager()
 143           pm.Add(ports, 'forwarded')
 144           self._forwarded_ports = pm.GetAllMappedPorts()
 145
 146     self._translate_configuration_files()
 147
 148     self._vm_health_check = _set_health_check_defaults(
 149         self._app_info_external.vm_health_check)
 150
 151   @property
 152   def application_root(self):
 153     """The directory containing the application e.g. "/home/user/myapp"."""
 154     return self._application_root
 155
 156   @property
 157   def application(self):
 158     return self._application
 159
 160   @property
 161   def partition(self):
 162     return 'dev'
 163
 164   @property
 165   def application_external_name(self):
 166     return self._app_info_external.application
 167
 168   @property
 169   def api_version(self):
 170     return self._api_version
 171
 172   @property
 173   def module_name(self):
 174     return self._module_name or appinfo.DEFAULT_MODULE
 175
 176   @property
 177   def major_version(self):
 178     return self._version
 179
 180   @property
 181   def minor_version(self):
 182     return self._minor_version_id
 183
 184   @property
 185   def version_id(self):
 186     if self.module_name == appinfo.DEFAULT_MODULE:
 187       return '%s.%s' % (
 188           self.major_version,
 189           self._minor_version_id)
 190     else:
 191       return '%s:%s.%s' % (
 192           self.module_name,
 193           self.major_version,
 194           self._minor_version_id)
 195
 196   @property
 197   def runtime(self):
 198     return self._runtime
 199
 200   @property
 201   def effective_runtime(self):
 202     return self._app_info_external.GetEffectiveRuntime()
 203
 204   @property
 205   def forwarded_ports(self):
 206     """A dictionary with forwarding rules as host_port => container_port."""
 207     return self._forwarded_ports
 208
 209   @property
 210   def threadsafe(self):
 211     return self._threadsafe
 212
 213   @property
 214   def basic_scaling(self):
 215     return self._basic_scaling
 216
 217   @property
 218   def manual_scaling(self):
 219     return self._manual_scaling
 220
 221   @property
 222   def automatic_scaling(self):
 223     return self._automatic_scaling
 224
 225   @property
 226   def normalized_libraries(self):
 227     return self._app_info_external.GetNormalizedLibraries()
 228
 229   @property
 230   def skip_files(self):
 231     return self._app_info_external.skip_files
 232
 233   @property
 234   def nobuild_files(self):
 235     return self._app_info_external.nobuild_files
 236
 237   @property
 238   def error_handlers(self):
 239     return self._app_info_external.error_handlers
 240
 241   @property
 242   def handlers(self):
 243     return self._app_info_external.handlers
 244
 245   @property
 246   def inbound_services(self):
 247     return self._app_info_external.inbound_services
 248
 249   @property
 250   def env_variables(self):
 251     return self._app_info_external.env_variables
 252
 253   @property
 254   def is_backend(self):
 255     return False
 256
 257   @property
 258   def config_path(self):
 259     return self._config_path
 260
 261   @property
 262   def vm_health_check(self):
 263     return self._vm_health_check
 264
 265   def check_for_updates(self):
 266     """Return any configuration changes since the last check_for_updates call.
 267
 268     Returns:
 269       A set containing the changes that occured. See the *_CHANGED module
 270       constants.
 271     """
 272     new_mtimes = self._get_mtimes(self._mtimes.keys())
 273     if new_mtimes == self._mtimes:
 274       return set()
 275
 276     try:
 277       app_info_external, files_to_check = self._parse_configuration(
 278           self._config_path)
 279     except Exception, e:
 280       failure_message = str(e)
 281       if failure_message != self._last_failure_message:
 282         logging.error('Configuration is not valid: %s', failure_message)
 283       self._last_failure_message = failure_message
 284       return set()
 285     self._last_failure_message = None
 286
 287     self._mtimes = self._get_mtimes(files_to_check)
 288
 289     for app_info_attribute, self_attribute in self._IMMUTABLE_PROPERTIES:
 290       app_info_value = getattr(app_info_external, app_info_attribute)
 291       self_value = getattr(self, self_attribute)
 292       if (app_info_value == self_value or
 293           app_info_value == getattr(self._app_info_external,
 294                                     app_info_attribute)):
 295         # Only generate a warning if the value is both different from the
 296         # immutable value *and* different from the last loaded value.
 297         continue
 298
 299       if isinstance(app_info_value, types.StringTypes):
 300         logging.warning('Restart the development module to see updates to "%s" '
 301                         '["%s" => "%s"]',
 302                         app_info_attribute,
 303                         self_value,
 304                         app_info_value)
 305       else:
 306         logging.warning('Restart the development module to see updates to "%s"',
 307                         app_info_attribute)
 308
 309     changes = set()
 310     if (app_info_external.GetNormalizedLibraries() !=
 311         self.normalized_libraries):
 312       changes.add(NORMALIZED_LIBRARIES_CHANGED)
 313     if app_info_external.skip_files != self.skip_files:
 314       changes.add(SKIP_FILES_CHANGED)
 315     if app_info_external.nobuild_files != self.nobuild_files:
 316       changes.add(NOBUILD_FILES_CHANGED)
 317     if app_info_external.handlers != self.handlers:
 318       changes.add(HANDLERS_CHANGED)
 319     if app_info_external.inbound_services != self.inbound_services:
 320       changes.add(INBOUND_SERVICES_CHANGED)
 321     if app_info_external.env_variables != self.env_variables:
 322       changes.add(ENV_VARIABLES_CHANGED)
 323     if app_info_external.error_handlers != self.error_handlers:
 324       changes.add(ERROR_HANDLERS_CHANGED)
 325
 326     self._app_info_external = app_info_external
 327     if changes:
 328       self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 329                                        range(18))
 330     return changes
 331
 332   @staticmethod
 333   def _get_mtimes(filenames):
 334     filename_to_mtime = {}
 335     for filename in filenames:
 336       try:
 337         filename_to_mtime[filename] = os.path.getmtime(filename)
 338       except OSError as e:
 339         # Ignore deleted includes.
 340         if e.errno != errno.ENOENT:
 341           raise
 342     return filename_to_mtime
 343
 344   def _parse_configuration(self, configuration_path):
 345     """Parse a configuration file (like app.yaml or appengine-web.xml).
 346
 347     Args:
 348       configuration_path: A string containing the full path of the yaml file
 349           containing the configuration for this module.
 350
 351     Returns:
 352       A tuple where the first element is the parsed appinfo.AppInfoExternal
 353       object and the second element is a list of the paths of the files that
 354       were used to produce it, namely the input configuration_path and any
 355       other file that was included from that one.
 356     """
 357     if self._is_java:
 358       config, files = self._parse_java_configuration(configuration_path)
 359     else:
 360       with open(configuration_path) as f:
 361         config, files = appinfo_includes.ParseAndReturnIncludePaths(f)
 362     if self._forced_app_id:
 363       config.application = self._forced_app_id
 364     return config, [configuration_path] + files
 365
 366   def _parse_java_configuration(self, app_engine_web_xml_path):
 367     """Parse appengine-web.xml and web.xml.
 368
 369     Args:
 370       app_engine_web_xml_path: A string containing the full path of the
 371           .../WEB-INF/appengine-web.xml file. The corresponding
 372           .../WEB-INF/web.xml file must also be present.
 373
 374     Returns:
 375       A tuple where the first element is the parsed appinfo.AppInfoExternal
 376       object and the second element is a list of the paths of the files that
 377       were used to produce it, namely the input appengine-web.xml file and the
 378       corresponding web.xml file.
 379     """
 380     with open(app_engine_web_xml_path) as f:
 381       app_engine_web_xml_str = f.read()
 382     web_inf_dir = os.path.dirname(app_engine_web_xml_path)
 383     web_xml_path = os.path.join(web_inf_dir, 'web.xml')
 384     with open(web_xml_path) as f:
 385       web_xml_str = f.read()
 386     has_jsps = False
 387     for _, _, filenames in os.walk(self.application_root):
 388       if any(f.endswith('.jsp') for f in filenames):
 389         has_jsps = True
 390         break
 391     app_yaml_str = yaml_translator.TranslateXmlToYamlForDevAppServer(
 392         app_engine_web_xml_str, web_xml_str, has_jsps, self.application_root)
 393     config = appinfo.LoadSingleAppInfo(app_yaml_str)
 394     return config, [app_engine_web_xml_path, web_xml_path]
 395
 396   def _translate_configuration_files(self):
 397     """Writes YAML equivalents of certain XML configuration files."""
 398     # For the most part we translate files in memory rather than writing out
 399     # translations. But since the task queue stub (taskqueue_stub.py)
 400     # reads queue.yaml directly rather than being configured with it, we need
 401     # to write a translation for the stub to find.
 402     # This means that we won't detect a change to the queue.xml, but we don't
 403     # currently have logic to react to changes to queue.yaml either.
 404     web_inf = os.path.join(self._application_root, 'WEB-INF')
 405     queue_xml_file = os.path.join(web_inf, 'queue.xml')
 406     if os.path.exists(queue_xml_file):
 407       appengine_generated = os.path.join(web_inf, 'appengine-generated')
 408       if not os.path.exists(appengine_generated):
 409         os.mkdir(appengine_generated)
 410       queue_yaml_file = os.path.join(appengine_generated, 'queue.yaml')
 411       with open(queue_xml_file) as f:
 412         queue_xml = f.read()
 413       queue_yaml = queue_xml_parser.GetQueueYaml(None, queue_xml)
 414       with open(queue_yaml_file, 'w') as f:
 415         f.write(queue_yaml)
 416
 417
 418 def _set_health_check_defaults(vm_health_check):
 419   """Sets default values for any missing attributes in VmHealthCheck.
 420
 421   These defaults need to be kept up to date with the production values in
 422   vm_health_check.cc
 423
 424   Args:
 425     vm_health_check: An instance of appinfo.VmHealthCheck or None.
 426
 427   Returns:
 428     An instance of appinfo.VmHealthCheck
 429   """
 430   if not vm_health_check:
 431     vm_health_check = appinfo.VmHealthCheck()
 432   for k, v in _HEALTH_CHECK_DEFAULTS.iteritems():
 433     if getattr(vm_health_check, k) is None:
 434       setattr(vm_health_check, k, v)
 435   return vm_health_check
 436
 437
 438 class BackendsConfiguration(object):
 439   """Stores configuration information for a backends.yaml file."""
 440
 441   def __init__(self, app_config_path, backend_config_path, app_id=None):
 442     """Initializer for BackendsConfiguration.
 443
 444     Args:
 445       app_config_path: A string containing the full path of the yaml file
 446           containing the configuration for this module.
 447       backend_config_path: A string containing the full path of the
 448           backends.yaml file containing the configuration for backends.
 449       app_id: A string that is the application id, or None if the application id
 450           from the yaml or xml file should be used.
 451     """
 452     self._update_lock = threading.RLock()
 453     self._base_module_configuration = ModuleConfiguration(
 454         app_config_path, app_id)
 455     backend_info_external = self._parse_configuration(
 456         backend_config_path)
 457
 458     self._backends_name_to_backend_entry = {}
 459     for backend in backend_info_external.backends or []:
 460       self._backends_name_to_backend_entry[backend.name] = backend
 461       self._changes = dict(
 462           (backend_name, set())
 463           for backend_name in self._backends_name_to_backend_entry)
 464
 465   @staticmethod
 466   def _parse_configuration(configuration_path):
 467     # TODO: It probably makes sense to catch the exception raised
 468     # by Parse() and re-raise it using a module-specific exception.
 469     with open(configuration_path) as f:
 470       return backendinfo.LoadBackendInfo(f)
 471
 472   def get_backend_configurations(self):
 473     return [BackendConfiguration(self._base_module_configuration, self, entry)
 474             for entry in self._backends_name_to_backend_entry.values()]
 475
 476   def check_for_updates(self, backend_name):
 477     """Return any configuration changes since the last check_for_updates call.
 478
 479     Args:
 480       backend_name: A str containing the name of the backend to be checked for
 481           updates.
 482
 483     Returns:
 484       A set containing the changes that occured. See the *_CHANGED module
 485       constants.
 486     """
 487     with self._update_lock:
 488       module_changes = self._base_module_configuration.check_for_updates()
 489       if module_changes:
 490         for backend_changes in self._changes.values():
 491           backend_changes.update(module_changes)
 492       changes = self._changes[backend_name]
 493       self._changes[backend_name] = set()
 494     return changes
 495
 496
 497 class BackendConfiguration(object):
 498   """Stores backend configuration information.
 499
 500   This interface is and must remain identical to ModuleConfiguration.
 501   """
 502
 503   def __init__(self, module_configuration, backends_configuration,
 504                backend_entry):
 505     """Initializer for BackendConfiguration.
 506
 507     Args:
 508       module_configuration: A ModuleConfiguration to use.
 509       backends_configuration: The BackendsConfiguration that tracks updates for
 510           this BackendConfiguration.
 511       backend_entry: A backendinfo.BackendEntry containing the backend
 512           configuration.
 513     """
 514     self._module_configuration = module_configuration
 515     self._backends_configuration = backends_configuration
 516     self._backend_entry = backend_entry
 517
 518     if backend_entry.dynamic:
 519       self._basic_scaling = appinfo.BasicScaling(
 520           max_instances=backend_entry.instances or 1)
 521       self._manual_scaling = None
 522     else:
 523       self._basic_scaling = None
 524       self._manual_scaling = appinfo.ManualScaling(
 525           instances=backend_entry.instances or 1)
 526     self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 527                                      range(18))
 528
 529   @property
 530   def application_root(self):
 531     """The directory containing the application e.g. "/home/user/myapp"."""
 532     return self._module_configuration.application_root
 533
 534   @property
 535   def application(self):
 536     return self._module_configuration.application
 537
 538   @property
 539   def partition(self):
 540     return self._module_configuration.partition
 541
 542   @property
 543   def application_external_name(self):
 544     return self._module_configuration.application_external_name
 545
 546   @property
 547   def api_version(self):
 548     return self._module_configuration.api_version
 549
 550   @property
 551   def module_name(self):
 552     return self._backend_entry.name
 553
 554   @property
 555   def major_version(self):
 556     return self._module_configuration.major_version
 557
 558   @property
 559   def minor_version(self):
 560     return self._minor_version_id
 561
 562   @property
 563   def version_id(self):
 564     return '%s:%s.%s' % (
 565         self.module_name,
 566         self.major_version,
 567         self._minor_version_id)
 568
 569   @property
 570   def runtime(self):
 571     return self._module_configuration.runtime
 572
 573   @property
 574   def effective_runtime(self):
 575     return self._module_configuration.effective_runtime
 576
 577   @property
 578   def forwarded_ports(self):
 579     return self._module_configuration.forwarded_ports
 580
 581   @property
 582   def threadsafe(self):
 583     return self._module_configuration.threadsafe
 584
 585   @property
 586   def basic_scaling(self):
 587     return self._basic_scaling
 588
 589   @property
 590   def manual_scaling(self):
 591     return self._manual_scaling
 592
 593   @property
 594   def automatic_scaling(self):
 595     return None
 596
 597   @property
 598   def normalized_libraries(self):
 599     return self._module_configuration.normalized_libraries
 600
 601   @property
 602   def skip_files(self):
 603     return self._module_configuration.skip_files
 604
 605   @property
 606   def nobuild_files(self):
 607     return self._module_configuration.nobuild_files
 608
 609   @property
 610   def error_handlers(self):
 611     return self._module_configuration.error_handlers
 612
 613   @property
 614   def handlers(self):
 615     if self._backend_entry.start:
 616       return [appinfo.URLMap(
 617           url='/_ah/start',
 618           script=self._backend_entry.start,
 619           login='admin')] + self._module_configuration.handlers
 620     return self._module_configuration.handlers
 621
 622   @property
 623   def inbound_services(self):
 624     return self._module_configuration.inbound_services
 625
 626   @property
 627   def env_variables(self):
 628     return self._module_configuration.env_variables
 629
 630   @property
 631   def is_backend(self):
 632     return True
 633
 634   @property
 635   def config_path(self):
 636     return self._module_configuration.config_path
 637
 638   @property
 639   def vm_health_check(self):
 640     return self._module_configuration.vm_health_check
 641
 642   def check_for_updates(self):
 643     """Return any configuration changes since the last check_for_updates call.
 644
 645     Returns:
 646       A set containing the changes that occured. See the *_CHANGED module
 647       constants.
 648     """
 649     changes = self._backends_configuration.check_for_updates(
 650         self._backend_entry.name)
 651     if changes:
 652       self._minor_version_id = ''.join(random.choice(string.digits) for _ in
 653                                        range(18))
 654     return changes
 655
 656
 657 class DispatchConfiguration(object):
 658   """Stores dispatcher configuration information."""
 659
 660   def __init__(self, config_path):
 661     self._config_path = config_path
 662     self._mtime = os.path.getmtime(self._config_path)
 663     self._process_dispatch_entries(self._parse_configuration(self._config_path))
 664
 665   @staticmethod
 666   def _parse_configuration(configuration_path):
 667     # TODO: It probably makes sense to catch the exception raised
 668     # by LoadSingleDispatch() and re-raise it using a module-specific exception.
 669     with open(configuration_path) as f:
 670       return dispatchinfo.LoadSingleDispatch(f)
 671
 672   def check_for_updates(self):
 673     mtime = os.path.getmtime(self._config_path)
 674     if mtime > self._mtime:
 675       self._mtime = mtime
 676       try:
 677         dispatch_info_external = self._parse_configuration(self._config_path)
 678       except Exception, e:
 679         failure_message = str(e)
 680         logging.error('Configuration is not valid: %s', failure_message)
 681         return
 682       self._process_dispatch_entries(dispatch_info_external)
 683
 684   def _process_dispatch_entries(self, dispatch_info_external):
 685     path_only_entries = []
 686     hostname_entries = []
 687     for entry in dispatch_info_external.dispatch:
 688       parsed_url = dispatchinfo.ParsedURL(entry.url)
 689       if parsed_url.host:
 690         hostname_entries.append(entry)
 691       else:
 692         path_only_entries.append((parsed_url, entry.module))
 693     if hostname_entries:
 694       logging.warning(
 695           'Hostname routing is not supported by the development server. The '
 696           'following dispatch entries will not match any requests:\n%s',
 697           '\n\t'.join(str(entry) for entry in hostname_entries))
 698     self._entries = path_only_entries
 699
 700   @property
 701   def dispatch(self):
 702     return self._entries
 703
 704
 705 class ApplicationConfiguration(object):
 706   """Stores application configuration information."""
 707
 708   def __init__(self, config_paths, app_id=None):
 709     """Initializer for ApplicationConfiguration.
 710
 711     Args:
 712       config_paths: A list of strings containing the paths to yaml files,
 713           or to directories containing them.
 714       app_id: A string that is the application id, or None if the application id
 715           from the yaml or xml file should be used.
 716     """
 717     self.modules = []
 718     self.dispatch = None
 719     # It's really easy to add a test case that passes in a string rather than
 720     # a list of strings, so guard against that.
 721     assert not isinstance(config_paths, basestring)
 722     config_paths = self._config_files_from_paths(config_paths)
 723     for config_path in config_paths:
 724       # TODO: add support for backends.xml and dispatch.xml here
 725       if (config_path.endswith('backends.yaml') or
 726           config_path.endswith('backends.yml')):
 727         # TODO: Reuse the ModuleConfiguration created for the app.yaml
 728         # instead of creating another one for the same file.
 729         app_yaml = config_path.replace('backends.y', 'app.y')
 730         self.modules.extend(
 731             BackendsConfiguration(
 732                 app_yaml, config_path, app_id).get_backend_configurations())
 733       elif (config_path.endswith('dispatch.yaml') or
 734             config_path.endswith('dispatch.yml')):
 735         if self.dispatch:
 736           raise errors.InvalidAppConfigError(
 737               'Multiple dispatch.yaml files specified')
 738         self.dispatch = DispatchConfiguration(config_path)
 739       else:
 740         module_configuration = ModuleConfiguration(config_path, app_id)
 741         self.modules.append(module_configuration)
 742     application_ids = set(module.application
 743                           for module in self.modules)
 744     if len(application_ids) > 1:
 745       raise errors.InvalidAppConfigError(
 746           'More than one application ID found: %s' %
 747           ', '.join(sorted(application_ids)))
 748
 749     self._app_id = application_ids.pop()
 750     module_names = set()
 751     for module in self.modules:
 752       if module.module_name in module_names:
 753         raise errors.InvalidAppConfigError('Duplicate module: %s' %
 754                                            module.module_name)
 755       module_names.add(module.module_name)
 756     if self.dispatch:
 757       if appinfo.DEFAULT_MODULE not in module_names:
 758         raise errors.InvalidAppConfigError(
 759             'A default module must be specified.')
 760       missing_modules = (
 761           set(module_name for _, module_name in self.dispatch.dispatch) -
 762           module_names)
 763       if missing_modules:
 764         raise errors.InvalidAppConfigError(
 765             'Modules %s specified in dispatch.yaml are not defined by a yaml '
 766             'file.' % sorted(missing_modules))
 767
 768   def _config_files_from_paths(self, config_paths):
 769     """Return a list of the configuration files found in the given paths.
 770
 771     For any path that is a directory, the returned list will contain the
 772     configuration files (app.yaml and optionally backends.yaml) found in that
 773     directory. If the directory is a Java app (contains a subdirectory
 774     WEB-INF with web.xml and application-web.xml files), then the returned
 775     list will contain the path to the application-web.xml file, which is treated
 776     as if it included web.xml. Paths that are not directories are added to the
 777     returned list as is.
 778
 779     Args:
 780       config_paths: a list of strings that are file or directory paths.
 781
 782     Returns:
 783       A list of strings that are file paths.
 784     """
 785     config_files = []
 786     for path in config_paths:
 787       config_files += (
 788           self._config_files_from_dir(path) if os.path.isdir(path) else [path])
 789     return config_files
 790
 791   def _config_files_from_dir(self, dir_path):
 792     """Return a list of the configuration files found in the given directory.
 793
 794     If the directory contains a subdirectory WEB-INF then we expect to find
 795     web.xml and application-web.xml in that subdirectory. The returned list
 796     will consist of the path to application-web.xml, which we treat as if it
 797     included web.xml.
 798
 799     Otherwise, we expect to find an app.yaml and optionally a backends.yaml,
 800     and we return those in the list.
 801
 802     Args:
 803       dir_path: a string that is the path to a directory.
 804
 805     Returns:
 806       A list of strings that are file paths.
 807     """
 808     web_inf = os.path.join(dir_path, 'WEB-INF')
 809     if java_supported() and os.path.isdir(web_inf):
 810       return self._config_files_from_web_inf_dir(web_inf)
 811     app_yamls = self._files_in_dir_matching(dir_path, ['app.yaml', 'app.yml'])
 812     if not app_yamls:
 813       or_web_inf = ' or a WEB-INF subdirectory' if java_supported() else ''
 814       raise errors.AppConfigNotFoundError(
 815           '"%s" is a directory but does not contain app.yaml or app.yml%s' %
 816           (dir_path, or_web_inf))
 817     backend_yamls = self._files_in_dir_matching(
 818         dir_path, ['backends.yaml', 'backends.yml'])
 819     return app_yamls + backend_yamls
 820
 821   def _config_files_from_web_inf_dir(self, web_inf):
 822     required = ['appengine-web.xml', 'web.xml']
 823     missing = [f for f in required
 824                if not os.path.exists(os.path.join(web_inf, f))]
 825     if missing:
 826       raise errors.AppConfigNotFoundError(
 827           'The "%s" subdirectory exists but is missing %s' %
 828           (web_inf, ' and '.join(missing)))
 829     return [os.path.join(web_inf, required[0])]
 830
 831   @staticmethod
 832   def _files_in_dir_matching(dir_path, names):
 833     abs_names = [os.path.join(dir_path, name) for name in names]
 834     files = [f for f in abs_names if os.path.exists(f)]
 835     if len(files) > 1:
 836       raise errors.InvalidAppConfigError(
 837           'Directory "%s" contains %s' % (dir_path, ' and '.join(names)))
 838     return files
 839
 840   @property
 841   def app_id(self):
 842     return self._app_id
 843
 844
 845 def get_app_error_file(module_configuration):
 846   """Returns application specific file to handle errors.
 847
 848   Dev AppServer only supports 'default' error code.
 849
 850   Args:
 851     module_configuration: ModuleConfiguration.
 852
 853   Returns:
 854       A string containing full path to error handler file or
 855       None if no 'default' error handler is specified.
 856   """
 857   for error_handler in module_configuration.error_handlers or []:
 858     if not error_handler.error_code or error_handler.error_code == 'default':
 859       return os.path.join(module_configuration.application_root,
 860                           error_handler.file)
 861   return None