python/google/appengine/datastore/datastore_stub_index.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Utilities for generating and updating index.yaml."""
  22
  23
  24
  25
  26
  27
  28
  29 __all__ = ['GenerateIndexFromHistory',
  30            'IndexYamlUpdater',
  31           ]
  32
  33 import os
  34 import logging
  35
  36 from google.appengine.api import apiproxy_stub_map
  37 from google.appengine.api import yaml_errors
  38 from google.appengine.datastore import datastore_index
  39
  40 import yaml
  41
  42
  43 AUTO_MARKER = '\n# AUTOGENERATED\n'
  44
  45
  46 AUTO_COMMENT = '''
  47 # This index.yaml is automatically updated whenever the dev_appserver
  48 # detects that a new type of query is run.  If you want to manage the
  49 # index.yaml file manually, remove the above marker line (the line
  50 # saying "# AUTOGENERATED").  If you want to manage some indexes
  51 # manually, move them above the marker line.  The index.yaml file is
  52 # automatically uploaded to the admin console when you next deploy
  53 # your application using appcfg.py.
  54 '''
  55
  56
  57 def GenerateIndexFromHistory(query_history,
  58                              all_indexes=None, manual_indexes=None):
  59   """Generate most of the text for index.yaml from the query history.
  60
  61   Args:
  62     query_history: Query history, a dict mapping query
  63     all_indexes: Optional datastore_index.IndexDefinitions instance
  64       representing all the indexes found in the input file.  May be None.
  65     manual_indexes: Optional datastore_index.IndexDefinitions instance
  66       containing indexes for which we should not generate output.  May be None.
  67
  68   Returns:
  69     A string representation that can safely be appended to an existing
  70     index.yaml file. Returns the empty string if it would generate no output.
  71   """
  72
  73
  74
  75
  76
  77   all_keys = datastore_index.IndexDefinitionsToKeys(all_indexes)
  78   manual_keys = datastore_index.IndexDefinitionsToKeys(manual_indexes)
  79
  80
  81   indexes = dict((key, 0) for key in all_keys - manual_keys)
  82
  83
  84   for query, count in query_history.iteritems():
  85     required, kind, ancestor, props = (
  86         datastore_index.CompositeIndexForQuery(query))
  87     if required:
  88       props = datastore_index.GetRecommendedIndexProperties(props)
  89       key = (kind, ancestor, props)
  90       if key not in manual_keys:
  91         if key in indexes:
  92           indexes[key] += count
  93         else:
  94           indexes[key] = count
  95
  96   if not indexes:
  97     return ''
  98
  99
 100
 101
 102   res = []
 103   for (kind, ancestor, props), count in sorted(indexes.iteritems()):
 104
 105     res.append('')
 106     res.append(datastore_index.IndexYamlForQuery(kind, ancestor, props))
 107
 108   res.append('')
 109   return '\n'.join(res)
 110
 111
 112 class IndexYamlUpdater(object):
 113   """Helper class for updating index.yaml.
 114
 115   This class maintains some state about the query history and the
 116   index.yaml file in order to minimize the number of times index.yaml
 117   is actually overwritten.
 118   """
 119
 120
 121   index_yaml_is_manual = False
 122   index_yaml_mtime = None
 123   last_history_size = 0
 124
 125   def __init__(self, root_path):
 126     """Constructor.
 127
 128     Args:
 129       root_path: Path to the app's root directory.
 130     """
 131     self.root_path = root_path
 132
 133   def UpdateIndexYaml(self, openfile=open):
 134     """Update index.yaml.
 135
 136     Args:
 137       openfile: Used for dependency injection.
 138
 139     We only ever write to index.yaml if either:
 140     - it doesn't exist yet; or
 141     - it contains an 'AUTOGENERATED' comment.
 142
 143     All indexes *before* the AUTOGENERATED comment will be written
 144     back unchanged.  All indexes *after* the AUTOGENERATED comment
 145     will be updated with the latest query counts (query counts are
 146     reset by --clear_datastore).  Indexes that aren't yet in the file
 147     will be appended to the AUTOGENERATED section.
 148
 149     We keep track of some data in order to avoid doing repetitive work:
 150     - if index.yaml is fully manual, we keep track of its mtime to
 151       avoid parsing it over and over;
 152     - we keep track of the number of keys in the history dict since
 153       the last time we updated index.yaml (or decided there was
 154       nothing to update).
 155     """
 156
 157
 158
 159
 160
 161     index_yaml_file = os.path.join(self.root_path, 'index.yaml')
 162
 163
 164     try:
 165       index_yaml_mtime = os.path.getmtime(index_yaml_file)
 166     except os.error:
 167       index_yaml_mtime = None
 168
 169
 170     index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime)
 171     self.index_yaml_mtime = index_yaml_mtime
 172
 173
 174     datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 175     query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength()
 176     history_changed = (query_ci_history_len != self.last_history_size)
 177     self.last_history_size = query_ci_history_len
 178
 179
 180     if not (index_yaml_changed or history_changed):
 181       logging.debug('No need to update index.yaml')
 182       return
 183
 184
 185     if self.index_yaml_is_manual and not index_yaml_changed:
 186       logging.debug('Will not update manual index.yaml')
 187       return
 188
 189
 190     if index_yaml_mtime is None:
 191       index_yaml_data = None
 192     else:
 193       try:
 194
 195
 196
 197         fh = openfile(index_yaml_file, 'rU')
 198       except IOError:
 199         index_yaml_data = None
 200       else:
 201         try:
 202           index_yaml_data = fh.read()
 203         finally:
 204           fh.close()
 205
 206
 207     self.index_yaml_is_manual = (index_yaml_data is not None and
 208                                  AUTO_MARKER not in index_yaml_data)
 209     if self.index_yaml_is_manual:
 210       logging.info('Detected manual index.yaml, will not update')
 211       return
 212
 213
 214
 215     if index_yaml_data is None:
 216       all_indexes = None
 217     else:
 218       try:
 219         all_indexes = datastore_index.ParseIndexDefinitions(index_yaml_data)
 220       except yaml_errors.EventListenerError, e:
 221
 222         logging.error('Error parsing %s:\n%s', index_yaml_file, e)
 223         return
 224       except Exception, err:
 225
 226         logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file,
 227                       err.__class__.__module__, err.__class__.__name__, err)
 228         return
 229
 230
 231     if index_yaml_data is None:
 232       manual_part, prev_automatic_part = 'indexes:\n', ''
 233       manual_indexes = None
 234     else:
 235       manual_part, prev_automatic_part = index_yaml_data.split(AUTO_MARKER, 1)
 236       if prev_automatic_part.startswith(AUTO_COMMENT):
 237         prev_automatic_part = prev_automatic_part[len(AUTO_COMMENT):]
 238
 239       try:
 240         manual_indexes = datastore_index.ParseIndexDefinitions(manual_part)
 241       except Exception, err:
 242         logging.error('Error parsing manual part of %s: %s',
 243                       index_yaml_file, err)
 244         return
 245
 246
 247     automatic_part = GenerateIndexFromHistory(datastore_stub.QueryHistory(),
 248                                               all_indexes, manual_indexes)
 249
 250
 251
 252     if (index_yaml_mtime is None and automatic_part == '' or
 253         automatic_part == prev_automatic_part):
 254       logging.debug('No need to update index.yaml')
 255       return
 256
 257
 258     try:
 259       fh = openfile(index_yaml_file, 'w')
 260     except IOError, err:
 261       logging.error('Can\'t write index.yaml: %s', err)
 262       return
 263
 264
 265     try:
 266       logging.info('Updating %s', index_yaml_file)
 267       fh.write(manual_part)
 268       fh.write(AUTO_MARKER)
 269       fh.write(AUTO_COMMENT)
 270       fh.write(automatic_part)
 271     finally:
 272       fh.close()
 273
 274
 275     try:
 276       self.index_yaml_mtime = os.path.getmtime(index_yaml_file)
 277     except os.error, err:
 278       logging.error('Can\'t stat index.yaml we just wrote: %s', err)
 279       self.index_yaml_mtime = None