python/google/appengine/tools/backends_conversion.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19 """Tool for converting Backends configuration to Modules configuration.
  20
  21 Uses existing backends.yaml and app.yaml files to create a separate
  22 <module-name>.yaml file for each module defined in backends.yaml.
  23 """
  24
  25 from __future__ import with_statement
  26
  27 import os
  28 import sys
  29 import warnings
  30
  31 from google.appengine.api import appinfo
  32 from google.appengine.api import backendinfo
  33
  34
  35 warnings.simplefilter('default')
  36
  37
  38 __all__ = [
  39     'ConvertBackendToModules',
  40 ]
  41
  42
  43 START_URI = '/_ah/start'
  44 LOGIN_ADMIN = 'admin'
  45 DEPRECATION_TEXT = ('The failfast option is deprecated for Modules. No '
  46                     'equivalent option will be set.')
  47 DYNAMIC_PROMPT_TEXT = """\
  48 Backend %s is marked dynamic.
  49 Dynamic backends should be converted to basic_scaling type.
  50 Basic scaling modules require an integer max instances value.
  51 Please provide the max_instances [default: 1]: """
  52 MAIN_ERR_MESSAGE = """\
  53 Backends and App Config filename arguments not passed
  54 in correctly. Can't complete operation.
  55 """
  56 PRINT_FILE_DELIMITER = ('=' * 80) + '\n' + ('=' * 80)
  57
  58
  59 def _ToYAMLDefault(appinfo_config):
  60   """Converts an app config to default (alphabetical by key) YAML string.
  61
  62   Args:
  63     appinfo_config: appinfo.AppInfoExternal object. Configuration object
  64       for either a module or app.yaml.
  65
  66   Returns:
  67     String containing YAML for the app config object.
  68   """
  69   return appinfo_config.ToYAML()
  70
  71
  72
  73 def ConvertBackendToModules(backend_config_filename,
  74                             app_config_filename,
  75                             _to_yaml_method=_ToYAMLDefault):
  76   """Creates Modules configuration using filenames of app and backend config.
  77
  78   Tries to write config to a file for each module defined.
  79
  80   Args:
  81     backend_config_filename: String; Relative path to backends.yaml passed in.
  82     app_config_filename: String; Relative path to app.yaml passed in.
  83     _to_yaml_method: A method which takes an appinfo.AppInfoExternal object and
  84       converts it to a YAML string. Defaults to _ToYAMLDefault which just calls
  85       ToYAML() on the object.
  86   """
  87   with open(backend_config_filename, 'r') as fh:
  88     backend_config = fh.read()
  89   with open(app_config_filename, 'r') as fh:
  90     app_config = fh.read()
  91
  92
  93   application_root, app_config_filename = os.path.split(
  94       os.path.abspath(app_config_filename))
  95   converted_backends = _ConvertBackendToModules(backend_config, app_config)
  96   for module_config in converted_backends:
  97     _MaybeWriteConfigToFile(module_config, application_root,
  98                             _to_yaml_method=_to_yaml_method)
  99
 100
 101 def _MaybeWriteConfigToFile(appinfo_config, application_root,
 102                             _to_yaml_method=_ToYAMLDefault):
 103   """Writes an app config to a file.
 104
 105   If the file already exists, prompts the user before saving. If the user
 106   does not wish to overwrite the file, prints the would-be file contents.
 107
 108   Args:
 109     appinfo_config: appinfo.AppInfoExternal object. Configuration object
 110       for either a module or app.yaml.
 111     application_root: String; an absolute path where the application to be
 112       deployed is located on the local filesystem.
 113     _to_yaml_method: A method which takes an appinfo.AppInfoExternal object and
 114       converts it to a YAML string. Defaults to _ToYAMLDefault which just calls
 115       ToYAML() on the object.
 116   """
 117   filename = '%s.yaml' % (appinfo_config.module.encode('ascii'),)
 118   filepath = os.path.join(application_root, filename)
 119
 120   contents = _to_yaml_method(appinfo_config)
 121   if os.path.exists(filepath):
 122     prompt = 'File %s exists. Overwrite? [y/N] ' % (filename,)
 123     result = raw_input(prompt).strip()
 124     if result != 'y':
 125       print 'File %s not written.' % (filename,)
 126       print 'Contents:'
 127       print PRINT_FILE_DELIMITER
 128       print contents
 129       print PRINT_FILE_DELIMITER
 130       return
 131
 132   with open(filepath, 'w') as fh:
 133     fh.write(contents)
 134
 135
 136
 137 def _ConvertBackendToModules(backend_config, app_config):
 138   """Creates Modules configuration using app and backend config.
 139
 140   Parses the app.yaml and backend.yaml contents into native AppInfoExternal
 141   and BackendInfoExternal objects and then creates an AppInfoExternal
 142   for each backend defined in backend_config.
 143
 144   Args:
 145     backend_config: String, the contents of backend.yaml.
 146     app_config: String, the contents of app.yaml.
 147
 148   Returns:
 149     A list of AppInfoExternal objects for each module.
 150   """
 151   backend_info = backendinfo.LoadBackendInfo(backend_config)
 152   app_yaml_config = appinfo.LoadSingleAppInfo(app_config)
 153   return [_ConvertBackendToModule(backend, app_yaml_config)
 154           for backend in backend_info.backends]
 155
 156
 157 def _ConvertBackendToModule(backend_entry, app_yaml_config):
 158   """Converts an individual backend to a module config.
 159
 160   Args:
 161     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 162       definition from backends.yaml.
 163     app_yaml_config: A appinfo.AppInfoExternal object. Contains parsed app.yaml.
 164
 165   Returns:
 166     An appinfo.AppInfoExternal object which is a copy of app.yaml patched with
 167       the backend definition.
 168   """
 169   result = _CopyAppInfo(app_yaml_config)
 170
 171
 172   _MaybeSetNotPublic(result, backend_entry)
 173
 174   _WarnFailFast(backend_entry)
 175
 176   _SetStart(result, backend_entry)
 177   _SetModule(result, backend_entry)
 178   _SetClass(result, backend_entry)
 179   _SetScalingType(result, backend_entry)
 180   return result
 181
 182
 183 def _CopyAppInfo(app_yaml_config):
 184   """Deep copy of parsed YAML config.
 185
 186   Casts native YAML object to string and then back again.
 187
 188   Args:
 189     app_yaml_config: A appinfo.AppInfoExternal object. Contains parsed app.yaml.
 190
 191   Returns:
 192     Deep copy of app_yaml_config.
 193   """
 194   as_yaml = app_yaml_config.ToYAML()
 195   return appinfo.LoadSingleAppInfo(as_yaml)
 196
 197
 198 def _MaybeSetNotPublic(target, backend_entry):
 199   """Attempts to set all handlers as login: admin if the backend is private.
 200
 201   Prompts user if this operation is desired before doing so. If the user
 202   declines, does nothing.
 203
 204   Args:
 205     target: A appinfo.AppInfoExternal object. Contains parsed app.yaml augmented
 206       by current backend info.
 207     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 208       definition from backends.yaml.
 209   """
 210   if backend_entry.public:
 211     return
 212
 213   prompt = ('Backend %s is marked private.\nWould you like to make all '
 214             'handlers \'login: admin\'? [y/N] ' % (backend_entry.name,))
 215   result = raw_input(prompt).strip()
 216   if result == 'y':
 217     for handler in target.handlers:
 218       handler.login = LOGIN_ADMIN
 219
 220
 221 def _WarnFailFast(backend_entry):
 222   """Warns if the deprecated failfast option is used in the backend.
 223
 224   Args:
 225     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 226       definition from backends.yaml.
 227   """
 228   if backend_entry.failfast:
 229     warnings.warn(DEPRECATION_TEXT, DeprecationWarning)
 230
 231
 232 def _RemoveStartHandler(app_yaml_config):
 233   """Removes a start handler from an application config if one is defined.
 234
 235   If multiple start handlers are defined, only the first would be used (since
 236   routing goes in order of first to last).
 237
 238   Args:
 239     app_yaml_config: A appinfo.AppInfoExternal object. Contains parsed app.yaml.
 240
 241   Returns:
 242     Either None, if there is no start handler or the removed appinfo.URLMap
 243       object containing the start handler info.
 244   """
 245   handlers = app_yaml_config.handlers
 246   start_handlers = []
 247   for handler in handlers:
 248     if handler.url == START_URI:
 249
 250       start_handlers.append(handler)
 251
 252   if start_handlers:
 253     for start_handler in start_handlers:
 254       handlers.remove(start_handler)
 255
 256     return start_handlers[0]
 257
 258
 259 def _SetStart(target, backend_entry):
 260   """Attempts to set a start handler for the target module.
 261
 262   This only gets set if there is a start script defined for the backend. If
 263   there was also a start handler in app.yaml, will copy this and use the
 264   existing handler, replacing the script with the one from the backend.
 265
 266   Args:
 267     target: A appinfo.AppInfoExternal object. Contains parsed app.yaml augmented
 268       by current backend info.
 269     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 270       definition from backends.yaml.
 271   """
 272   if backend_entry.start is None:
 273     return
 274
 275   start_handler = _RemoveStartHandler(target)
 276   if start_handler is None:
 277     start_handler = appinfo.URLMap(url=START_URI, login=LOGIN_ADMIN)
 278
 279   start_handler.script = backend_entry.start
 280   target.handlers.insert(0, start_handler)
 281
 282
 283 def _SetModule(target, backend_entry):
 284   """Sets module name to backend name.
 285
 286   Args:
 287     target: A appinfo.AppInfoExternal object. Contains parsed app.yaml augmented
 288       by current backend info.
 289     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 290       definition from backends.yaml.
 291   """
 292   target.module = backend_entry.name
 293
 294
 295 def _SetClass(target, backend_entry):
 296   """Sets module instance class to backend instance class.
 297
 298   If there was no instance class defined on the backend, does nothing.
 299
 300   Args:
 301     target: A appinfo.AppInfoExternal object. Contains parsed app.yaml augmented
 302       by current backend info.
 303     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 304       definition from backends.yaml.
 305   """
 306   curr_class = backend_entry.get_class()
 307   if curr_class is not None:
 308     target.instance_class = curr_class
 309
 310
 311 def _SetManualScaling(target, backend_entry):
 312   """Sets scaling type to manual with specified number of instances.
 313
 314   If instances not defined in backend does nothing. Otherwise, sets the manual
 315   scaling field to use the number of instances specified.
 316
 317   Args:
 318     target: A appinfo.AppInfoExternal object. Contains parsed app.yaml augmented
 319       by current backend info.
 320     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 321       definition from backends.yaml.
 322   """
 323   instances = backend_entry.instances
 324   if instances is not None:
 325     target.manual_scaling = appinfo.ManualScaling(instances=instances)
 326
 327
 328 def _GetInstances(name):
 329   """Gets a positive number of instances from the user.
 330
 331   Uses the DYNAMIC_PROMPT_TEXT to prompt the user. Accepts no
 332   input to mean the default value of 1.
 333
 334   Args:
 335     name: String, name of module.
 336
 337   Returns:
 338     Integer parsed from user input, 1 if empty input or None if the input was
 339       not a positive integer.
 340   """
 341   prompt = DYNAMIC_PROMPT_TEXT % (name,)
 342   result = raw_input(prompt).strip()
 343   if result == '':
 344     return 1
 345
 346   max_instances = -1
 347   try:
 348     max_instances = int(result)
 349   except (TypeError, ValueError):
 350     pass
 351
 352   if max_instances <= 0:
 353     print 'Invalid max_instances value: %r' % (result,)
 354     return
 355
 356   return max_instances
 357
 358
 359 def _SetScalingType(target, backend_entry):
 360   """Sets the scaling type of the modules based on the backend.
 361
 362   If dynamic, sets scaling type to Basic and passes the number of instances if
 363   set in the backends config. If not dynamic but instances set, calls to
 364   _SetManualScaling. If neither dynamic or instances set, does nothing.
 365
 366   Args:
 367     target: A appinfo.AppInfoExternal object. Contains parsed app.yaml augmented
 368       by current backend info.
 369     backend_entry: A backendinfo.BackendEntry object. Contains a parsed backend
 370       definition from backends.yaml.
 371   """
 372   if not (backend_entry.dynamic or backend_entry.instances):
 373     return
 374
 375
 376   if not backend_entry.dynamic:
 377     _SetManualScaling(target, backend_entry)
 378     return
 379
 380   if backend_entry.instances:
 381     max_instances = backend_entry.instances
 382   else:
 383     max_instances = _GetInstances(backend_entry.name)
 384
 385   if max_instances:
 386     target.basic_scaling = appinfo.BasicScaling(max_instances=max_instances)
 387
 388
 389 def MakeParser(prog):
 390   """Create an argument parser.
 391
 392   Args:
 393     prog: The name of the program to use when outputting help text.
 394
 395   Returns:
 396     An argparse.ArgumentParser built to specification.
 397   """
 398
 399
 400
 401   import argparse
 402   parser = argparse.ArgumentParser(prog=prog)
 403   parser.add_argument('backend_config_filename', nargs=1,
 404                       help='Path to backends.yaml for application.')
 405   parser.add_argument('app_config_filename', nargs=1,
 406                       help='Path to app.yaml for application.')
 407   return parser
 408
 409
 410 def main(argv):
 411   parser = MakeParser(argv[0])
 412   args = parser.parse_args(argv[1:])
 413
 414   backend_config_filename_args = getattr(args, 'backend_config_filename', [])
 415   app_config_filename_args = getattr(args, 'app_config_filename', [])
 416   if (len(backend_config_filename_args) != 1 or
 417       len(app_config_filename_args) != 1):
 418     print >>sys.stderr, MAIN_ERR_MESSAGE
 419     return 1
 420
 421   ConvertBackendToModules(backend_config_filename_args[0],
 422                           app_config_filename_args[0])
 423   return 0
 424
 425
 426 if __name__ == '__main__':
 427   sys.exit(main(sys.argv))