python/google/appengine/api/conf.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """A library for managing flags-like configuration that update dynamically.
  22 """
  23
  24
  25
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 try:
  32   from google.appengine.api import memcache
  33   from google.appengine.ext import db
  34   from google.appengine.api import validation
  35   from google.appengine.api import yaml_object
  36 except:
  37   from google.appengine.api import memcache
  38   from google.appengine.ext import db
  39   from google.appengine.ext import validation
  40   from google.appengine.ext import yaml_object
  41
  42
  43
  44
  45
  46 DATASTORE_DEADLINE = 1.5
  47
  48
  49 RESERVED_MARKER = 'ah__conf__'
  50
  51
  52
  53 NAMESPACE = '_' + RESERVED_MARKER
  54
  55 CONFIG_KIND = '_AppEngine_Config'
  56
  57 ACTIVE_KEY_NAME = 'active'
  58
  59 FILENAMES = ['conf.yaml', 'conf.yml']
  60
  61 PARAMETERS = 'parameters'
  62
  63
  64 PARAMETER_NAME_REGEX = '[a-zA-Z][a-zA-Z0-9_]*'
  65
  66
  67 _cached_config = None
  68
  69
  70 class Config(db.Expando):
  71   """The representation of a config in the datastore and memcache."""
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82   ah__conf__version = db.IntegerProperty(default=0, required=True)
  83
  84   @classmethod
  85   def kind(cls):
  86     """Override the kind name to prevent collisions with users."""
  87     return CONFIG_KIND
  88
  89   def ah__conf__load_from_yaml(self, parsed_config):
  90     """Loads all the params from a YAMLConfiguration into expando fields.
  91
  92     We set these expando properties with a special name prefix 'p_' to
  93     keep them separate from the static attributes of Config.  That way we
  94     don't have to check elsewhere to make sure the user doesn't stomp on
  95     our built in properties.
  96
  97     Args:
  98       parse_config: A YAMLConfiguration.
  99     """
 100     for key, value in parsed_config.parameters.iteritems():
 101       setattr(self, key, value)
 102
 103
 104 class _ValidParameterName(validation.Validator):
 105   """Validator to check if a value is a valid config parameter name.
 106
 107   We only allow valid python attribute names without leading underscores
 108   that also do not collide with reserved words in the datastore models.
 109   """
 110   def __init__(self):
 111     self.regex = validation.Regex(PARAMETER_NAME_REGEX)
 112
 113   def Validate(self, value, key):
 114     """Check that all parameter names are valid.
 115
 116     This is used as a validator when parsing conf.yaml.
 117
 118     Args:
 119       value: the value to check.
 120       key: A description of the context for which this value is being
 121       validated.
 122
 123     Returns:
 124       The validated value.
 125     """
 126     value = self.regex.Validate(value, key)
 127
 128     try:
 129       db.check_reserved_word(value)
 130     except db.ReservedWordError:
 131       raise validation.ValidationError(
 132           'The config parameter name %.100r is reserved by db.Model see: '
 133           'https://developers.google.com/appengine/docs/python/datastore/'
 134           'modelclass#Disallowed_Property_Names for details.' % value)
 135
 136     if value.startswith(RESERVED_MARKER):
 137       raise validation.ValidationError(
 138           'The config parameter name %.100r is reserved, as are all names '
 139           'beginning with \'%s\', please choose a different name.' % (
 140               value, RESERVED_MARKER))
 141
 142     return value
 143
 144
 145 class _Scalar(validation.Validator):
 146   """Validator to check if a value is a simple scalar type.
 147
 148   We only allow scalars that are well supported by both the datastore and YAML.
 149   """
 150   ALLOWED_PARAMETER_VALUE_TYPES = frozenset(
 151       [bool, int, long, float, str, unicode])
 152
 153   def Validate(self, value, key):
 154     """Check that all parameters are scalar values.
 155
 156     This is used as a validator when parsing conf.yaml
 157
 158     Args:
 159       value: the value to check.
 160       key: the name of parameter corresponding to this value.
 161
 162     Returns:
 163       We just return value unchanged.
 164     """
 165     if type(value) not in self.ALLOWED_PARAMETER_VALUE_TYPES:
 166       raise validation.ValidationError(
 167           'Expected scalar value for parameter: %s, but found %.100r which '
 168           'is type %s' % (key, value, type(value).__name__))
 169
 170     return value
 171
 172
 173 class _ParameterDict(validation.ValidatedDict):
 174   """This class validates the parameters dictionary in YAMLConfiguration.
 175
 176   Keys must look like non-private python identifiers and values
 177   must be a supported scalar.  See the class comment for YAMLConfiguration.
 178   """
 179   KEY_VALIDATOR = _ValidParameterName()
 180   VALUE_VALIDATOR = _Scalar()
 181
 182
 183 class YAMLConfiguration(validation.Validated):
 184   """This class describes the structure of a conf.yaml file.
 185
 186   At the top level the file should have a params attribue which is a mapping
 187   from strings to scalars.  For example:
 188
 189   parameters:
 190     background_color: 'red'
 191     message_size: 1024
 192     boolean_valued_param: true
 193   """
 194   ATTRIBUTES = {PARAMETERS: _ParameterDict}
 195
 196
 197 def LoadSingleConf(stream):
 198   """Load a conf.yaml file or string and return a YAMLConfiguration object.
 199
 200   Args:
 201     stream: a file object corresponding to a conf.yaml file, or its contents
 202       as a string.
 203
 204   Returns:
 205     A YAMLConfiguration instance
 206   """
 207   return yaml_object.BuildSingleObject(YAMLConfiguration, stream)
 208
 209
 210
 211
 212 def _find_yaml_path():
 213   """Traverse directory trees to find conf.yaml file.
 214
 215   Begins with the current working direcotry and then moves up the
 216   directory structure until the file is found..
 217
 218   Returns:
 219     the path of conf.yaml file or None if not found.
 220   """
 221   current, last = os.getcwd(), None
 222   while current != last:
 223     for yaml_name in FILENAMES:
 224       yaml_path = os.path.join(current, yaml_name)
 225       if os.path.exists(yaml_path):
 226         return yaml_path
 227     last = current
 228     current, last = os.path.dirname(current), current
 229
 230   return None
 231
 232
 233 def _fetch_from_local_file(pathfinder=_find_yaml_path, fileopener=open):
 234   """Get the configuration that was uploaded with this version.
 235
 236   Args:
 237     pathfinder: a callable to use for finding the path of the conf.yaml
 238     file.  This is only for use in testing.
 239     fileopener: a callable to use for opening a named file.  This is
 240     only for use in testing.
 241
 242   Returns:
 243     A config class instance for the options that were uploaded.  If there
 244     is no config file, return None
 245   """
 246
 247
 248   yaml_path = pathfinder()
 249   if yaml_path:
 250     config = Config()
 251     config.ah__conf__load_from_yaml(LoadSingleConf(fileopener(yaml_path)))
 252     logging.debug('Loaded conf parameters from conf.yaml.')
 253     return config
 254
 255   return None
 256
 257
 258 def _get_active_config_key(app_version):
 259   """Generate the key for the active config record belonging to app_version.
 260
 261   Args:
 262     app_version: the major version you want configuration data for.
 263
 264   Returns:
 265     The key for the active Config record for the given app_version.
 266   """
 267   return db.Key.from_path(
 268       CONFIG_KIND,
 269       '%s/%s' % (app_version, ACTIVE_KEY_NAME),
 270       namespace=NAMESPACE)
 271
 272
 273 def _fetch_latest_from_datastore(app_version):
 274   """Get the latest configuration data for this app-version from the datastore.
 275
 276   Args:
 277     app_version: the major version you want configuration data for.
 278
 279   Side Effects:
 280     We populate memcache with whatever we find in the datastore.
 281
 282   Returns:
 283     A config class instance for most recently set options or None if the
 284     query could not complete due to a datastore exception.
 285   """
 286
 287
 288
 289
 290
 291   rpc = db.create_rpc(deadline=DATASTORE_DEADLINE,
 292                       read_policy=db.EVENTUAL_CONSISTENCY)
 293   key = _get_active_config_key(app_version)
 294   config = None
 295   try:
 296     config = Config.get(key, rpc=rpc)
 297     logging.debug('Loaded most recent conf data from datastore.')
 298   except:
 299     logging.warning('Tried but failed to fetch latest conf data from the '
 300                     'datastore.')
 301
 302   if config:
 303     memcache.set(app_version, db.model_to_protobuf(config).Encode(),
 304                  namespace=NAMESPACE)
 305     logging.debug('Wrote most recent conf data into memcache.')
 306
 307   return config
 308
 309
 310 def _fetch_latest_from_memcache(app_version):
 311   """Get the latest configuration data for this app-version from memcache.
 312
 313   Args:
 314     app_version: the major version you want configuration data for.
 315
 316   Returns:
 317     A Config class instance for most recently set options or None if none
 318     could be found in memcache.
 319   """
 320   proto_string = memcache.get(app_version, namespace=NAMESPACE)
 321   if proto_string:
 322     logging.debug('Loaded most recent conf data from memcache.')
 323     return db.model_from_protobuf(proto_string)
 324
 325   logging.debug('Tried to load conf data from memcache, but found nothing.')
 326   return None
 327
 328
 329 def _inspect_environment():
 330   """Return relevant information from the cgi environment.
 331
 332   This is mostly split out to simplify testing.
 333
 334   Returns:
 335     A tuple: (app_version, conf_version, development)
 336     app_version: the major version of the current application.
 337     conf_version: the current configuration version.
 338     development: a boolean, True if we're running under devappserver.
 339   """
 340   app_version = os.environ['CURRENT_VERSION_ID'].rsplit('.', 1)[0]
 341   conf_version = int(os.environ.get('CURRENT_CONFIGURATION_VERSION', '0'))
 342   development = os.environ.get('SERVER_SOFTWARE', '').startswith('Development/')
 343   return (app_version, conf_version, development)
 344
 345
 346 def refresh():
 347   """Update the local config cache from memcache/datastore.
 348
 349   Normally configuration parameters are only refreshed at the start of a
 350   new request.  If you have a very long running request, or you just need
 351   the freshest data for some reason, you can call this function to force
 352   a refresh.
 353   """
 354   app_version, _, _ = _inspect_environment()
 355
 356
 357
 358
 359   global _cached_config
 360
 361   new_config = _fetch_latest_from_memcache(app_version)
 362
 363   if not new_config:
 364     new_config = _fetch_latest_from_datastore(app_version)
 365
 366   if new_config:
 367     _cached_config = new_config
 368
 369
 370 def _new_request():
 371   """Test if this is the first call to this function in the current request.
 372
 373   This function will return True exactly once for each request
 374   Subsequent calls in the same request will return False.
 375
 376   Returns:
 377     True if this is the first call in a given request, False otherwise.
 378   """
 379   if RESERVED_MARKER in os.environ:
 380     return False
 381
 382   os.environ[RESERVED_MARKER] = RESERVED_MARKER
 383   return True
 384
 385
 386 def _get_config():
 387   """Check if the current cached config is stale, and if so update it."""
 388
 389
 390
 391
 392
 393
 394
 395   app_version, current_config_version, development = _inspect_environment()
 396
 397   global _cached_config
 398
 399   if (development and _new_request()) or not _cached_config:
 400     _cached_config = _fetch_from_local_file() or Config()
 401
 402   if _cached_config.ah__conf__version < current_config_version:
 403     newconfig = _fetch_latest_from_memcache(app_version)
 404     if not newconfig or newconfig.ah__conf__version < current_config_version:
 405       newconfig = _fetch_latest_from_datastore(app_version)
 406
 407
 408
 409     _cached_config = newconfig or _cached_config
 410
 411   return _cached_config
 412
 413
 414 def get(name, default=None):
 415   """Get the value of a configuration parameter.
 416
 417   This function is guaranteed to return the same value for every call
 418   during a single request.
 419
 420   Args:
 421     name: The name of the configuration parameter you want a value for.
 422     default: A default value to return if the named parameter doesn't exist.
 423
 424   Returns:
 425     The string value of the configuration parameter.
 426   """
 427   return getattr(_get_config(), name, default)
 428
 429
 430 def get_all():
 431   """Return an object with an attribute for each conf parameter.
 432
 433   Returns:
 434     An object with an attribute for each conf parameter.
 435   """
 436   return _get_config()