python/google/appengine/api/appinfo_includes.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Used to parse app.yaml files while following builtins/includes directives."""
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34 import logging
  35 import os
  36
  37 from google.appengine.api import appinfo
  38 from google.appengine.api import appinfo_errors
  39 from google.appengine.ext import builtins
  40
  41
  42 class IncludeFileNotFound(Exception):
  43   """Raised if a specified include file cannot be found on disk."""
  44
  45
  46 def Parse(appinfo_file, open_fn=open):
  47   """Parse an AppYaml file and merge referenced includes and builtins.
  48
  49   Args:
  50     appinfo_file: an opened file, for example the result of open('app.yaml').
  51     open_fn: a function to open included files.
  52
  53   Returns:
  54     The parsed appinfo.AppInfoExternal object.
  55   """
  56   appyaml, _ = ParseAndReturnIncludePaths(appinfo_file, open_fn)
  57   return appyaml
  58
  59
  60 def ParseAndReturnIncludePaths(appinfo_file, open_fn=open):
  61   """Parse an AppYaml file and merge referenced includes and builtins.
  62
  63   Args:
  64     appinfo_file: an opened file, for example the result of open('app.yaml').
  65     open_fn: a function to open included files.
  66
  67   Returns:
  68     A tuple where the first element is the parsed appinfo.AppInfoExternal
  69     object and the second element is a list of the absolute paths of the
  70     included files, in no particular order.
  71   """
  72   try:
  73     appinfo_path = appinfo_file.name
  74     if not os.path.isfile(appinfo_path):
  75       raise Exception('Name defined by appinfo_file does not appear to be a '
  76                       'valid file: %s' % appinfo_path)
  77   except AttributeError:
  78     raise Exception('File object passed to ParseAndMerge does not define '
  79                     'attribute "name" as as full file path.')
  80
  81   appyaml = appinfo.LoadSingleAppInfo(appinfo_file)
  82   appyaml, include_paths = _MergeBuiltinsIncludes(appinfo_path, appyaml,
  83                                                   open_fn)
  84
  85
  86   if not appyaml.handlers:
  87
  88     if appyaml.vm:
  89       appyaml.handlers = [appinfo.URLMap(url='.*', script='PLACEHOLDER')]
  90     else:
  91       raise appinfo_errors.MissingURLMapping(
  92           'No URLMap entries found in application configuration')
  93   if len(appyaml.handlers) > appinfo.MAX_URL_MAPS:
  94     raise appinfo_errors.TooManyURLMappings(
  95         'Found more than %d URLMap entries in application configuration' %
  96         appinfo.MAX_URL_MAPS)
  97   if appyaml.runtime == 'python27' and appyaml.threadsafe:
  98     for handler in appyaml.handlers:
  99       if (handler.script and (handler.script.endswith('.py') or
 100                               '/' in handler.script)):
 101         raise appinfo_errors.ThreadsafeWithCgiHandler(
 102             'Threadsafe cannot be enabled with CGI handler: %s' %
 103             handler.script)
 104
 105   return appyaml, include_paths
 106
 107
 108 def _MergeBuiltinsIncludes(appinfo_path, appyaml, open_fn=open):
 109   """Merges app.yaml files from builtins and includes directives in appyaml.
 110
 111   Args:
 112     appinfo_path: the application directory.
 113     appyaml: the yaml file to obtain builtins and includes directives from.
 114     open_fn: file opening function to pass to _ResolveIncludes, used when
 115              reading yaml files.
 116
 117   Returns:
 118     A tuple where the first element is the modified appyaml object
 119     incorporating the referenced yaml files, and the second element is a list
 120     of the absolute paths of the included files, in no particular order.
 121   """
 122
 123
 124
 125   if not appyaml.builtins:
 126     appyaml.builtins = [appinfo.BuiltinHandler(default='on')]
 127
 128   else:
 129     if not appinfo.BuiltinHandler.IsDefined(appyaml.builtins, 'default'):
 130       appyaml.builtins.append(appinfo.BuiltinHandler(default='on'))
 131
 132
 133
 134   runtime_for_including = appyaml.runtime
 135   if runtime_for_including == 'vm':
 136     runtime_for_including = appyaml.vm_settings.get('vm_runtime', 'python27')
 137   aggregate_appinclude, include_paths = (
 138       _ResolveIncludes(appinfo_path,
 139                        appinfo.AppInclude(builtins=appyaml.builtins,
 140                                           includes=appyaml.includes),
 141                        os.path.dirname(appinfo_path),
 142                        runtime_for_including,
 143                        open_fn=open_fn))
 144
 145   return (
 146       appinfo.AppInclude.MergeAppYamlAppInclude(appyaml,
 147                                                 aggregate_appinclude),
 148       include_paths)
 149
 150
 151 def _ResolveIncludes(included_from, app_include, basepath, runtime, state=None,
 152                      open_fn=open):
 153   """Recursively includes all encountered builtins/includes directives.
 154
 155   This function takes an initial AppInclude object specified as a parameter
 156   and recursively evaluates every builtins/includes directive in the passed
 157   in AppInclude and any files they reference.  The sole output of the function
 158   is an AppInclude object that is the result of merging all encountered
 159   AppInclude objects.  This must then be merged with the root AppYaml object.
 160
 161   Args:
 162     included_from: file that included file was included from.
 163     app_include: the AppInclude object to resolve.
 164     basepath: application basepath.
 165     runtime: name of the runtime.
 166     state: contains the list of included and excluded files as well as the
 167            directives of all encountered AppInclude objects.
 168     open_fn: file opening function udes, used when reading yaml files.
 169
 170   Returns:
 171     A two-element tuple where the first element is the AppInclude object merged
 172     from following all builtins/includes defined in provided AppInclude object;
 173     and the second element is a list of the absolute paths of the included
 174     files, in no particular order.
 175
 176   Raises:
 177     IncludeFileNotFound: if file specified in an include statement cannot be
 178       resolved to an includeable file (result from _ResolvePath is False).
 179   """
 180
 181   class RecurseState(object):
 182
 183
 184
 185
 186
 187     def __init__(self):
 188       self.includes = {}
 189       self.excludes = {}
 190       self.aggregate_appinclude = appinfo.AppInclude()
 191
 192
 193   if not state:
 194     state = RecurseState()
 195
 196
 197   appinfo.AppInclude.MergeAppIncludes(state.aggregate_appinclude, app_include)
 198
 199
 200   includes_list = _ConvertBuiltinsToIncludes(included_from, app_include,
 201                                              state, runtime)
 202
 203
 204   includes_list.extend(app_include.includes or [])
 205
 206
 207   for i in includes_list:
 208     inc_path = _ResolvePath(included_from, i, basepath)
 209     if not inc_path:
 210       raise IncludeFileNotFound('File %s listed in includes directive of %s '
 211                                 'could not be found.' % (i, included_from))
 212
 213     if inc_path in state.excludes:
 214       logging.warning('%s already disabled by %s but later included by %s',
 215                       inc_path, state.excludes[inc_path], included_from)
 216     elif not inc_path in state.includes:
 217       state.includes[inc_path] = included_from
 218       yaml_file = open_fn(inc_path, 'r')
 219       try:
 220         inc_yaml = appinfo.LoadAppInclude(yaml_file)
 221         _ResolveIncludes(inc_path, inc_yaml, basepath, runtime, state=state,
 222                          open_fn=open_fn)
 223       except appinfo_errors.EmptyConfigurationFile:
 224
 225         if not os.path.basename(os.path.dirname(inc_path)) == 'default':
 226           logging.warning('Nothing to include in %s', inc_path)
 227
 228
 229   return state.aggregate_appinclude, state.includes.keys()
 230
 231
 232 def _ConvertBuiltinsToIncludes(included_from, app_include, state, runtime):
 233   """Converts builtins directives to includes directives.
 234
 235   Moves all builtins directives in app_include into the includes
 236   directives list.  Modifies class excludes dict if any builtins are
 237   switched off.  Class includes dict is used to determine if an excluded
 238   builtin was previously included.
 239
 240   Args:
 241     included_from: file that builtin directive was found in
 242     app_include: the AppInclude object currently being processed.
 243     state: contains the list of included and excluded files as well as the
 244            directives of all encountered AppInclude objects.
 245     runtime: name of the runtime.
 246
 247   Returns:
 248     list of the absolute paths to the include files for builtins where
 249     "x: on" directive was specified, e.g. "builtins:\n  default: on" ->
 250     ['/google/appengine/ext/builtins/default/include.yaml']
 251   """
 252   includes_list = []
 253   if app_include.builtins:
 254     builtins_list = appinfo.BuiltinHandler.ListToTuples(app_include.builtins)
 255     for builtin_name, on_or_off in builtins_list:
 256
 257       if not on_or_off:
 258         continue
 259
 260
 261       yaml_path = builtins.get_yaml_path(builtin_name, runtime)
 262
 263       if on_or_off == 'on':
 264         includes_list.append(yaml_path)
 265       elif on_or_off == 'off':
 266         if yaml_path in state.includes:
 267           logging.warning('%s already included by %s but later disabled by %s',
 268                           yaml_path, state.includes[yaml_path], included_from)
 269         state.excludes[yaml_path] = included_from
 270       else:
 271         logging.error('Invalid state for AppInclude object loaded from %s; '
 272                       'builtins directive "%s: %s" ignored.',
 273                       included_from, builtin_name, on_or_off)
 274
 275   return includes_list
 276
 277
 278 def _ResolvePath(included_from, included_path, basepath):
 279   """Gets the absolute path of the file to be included.
 280
 281   Resolves in the following order:
 282   - absolute path or relative to working directory
 283     (path as specified resolves to a file)
 284   - relative to basepath
 285     (basepath + path resolves to a file)
 286   - relative to file it was included from
 287     (included_from + included_path resolves to a file)
 288
 289   Args:
 290     included_from: absolute path of file that included_path was included from.
 291     included_path: file string from includes directive.
 292     basepath: the application directory.
 293
 294   Returns:
 295     absolute path of the first file found for included_path or ''.
 296   """
 297
 298
 299
 300
 301
 302
 303
 304   path = os.path.join(os.path.dirname(included_from), included_path)
 305   if not _IsFileOrDirWithFile(path):
 306
 307     path = os.path.join(basepath, included_path)
 308     if not _IsFileOrDirWithFile(path):
 309
 310       path = included_path
 311       if not _IsFileOrDirWithFile(path):
 312         return ''
 313
 314   if os.path.isfile(path):
 315     return os.path.normcase(os.path.abspath(path))
 316
 317   return os.path.normcase(os.path.abspath(os.path.join(path, 'include.yaml')))
 318
 319
 320 def _IsFileOrDirWithFile(path):
 321   """Determine if a path is a file or a directory with an appropriate file."""
 322   return os.path.isfile(path) or (
 323       os.path.isdir(path) and
 324       os.path.isfile(os.path.join(path, 'include.yaml')))