python/google/appengine/datastore/datastore_stub_index.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Utilities for generating and updating index.yaml."""
  22
  23 from __future__ import with_statement
  24
  25
  26
  27
  28
  29
  30
  31 __all__ = ['GenerateIndexFromHistory',
  32            'IndexYamlUpdater',
  33           ]
  34
  35 import logging
  36 import os
  37
  38 from google.appengine.api import apiproxy_stub_map
  39 from google.appengine.api import validation
  40 from google.appengine.api import yaml_errors
  41 from google.appengine.datastore import datastore_index
  42 from google.appengine.datastore import datastore_index_xml
  43
  44 import yaml
  45
  46
  47 AUTO_MARKER = '\n# AUTOGENERATED\n'
  48
  49
  50 AUTO_COMMENT = '''
  51 # This index.yaml is automatically updated whenever the dev_appserver
  52 # detects that a new type of query is run.  If you want to manage the
  53 # index.yaml file manually, remove the above marker line (the line
  54 # saying "# AUTOGENERATED").  If you want to manage some indexes
  55 # manually, move them above the marker line.  The index.yaml file is
  56 # automatically uploaded to the admin console when you next deploy
  57 # your application using appcfg.py.
  58 '''
  59
  60
  61 def GenerateIndexDictFromHistory(query_history,
  62                                  all_indexes=None, manual_indexes=None):
  63   """Generate a dict of automatic index entries from the query history.
  64
  65   Args:
  66     query_history: Query history, a dict mapping datastore_pb.Query to a count
  67       of the number of times that query has been issued.
  68     all_indexes: Optional datastore_index.IndexDefinitions instance
  69       representing all the indexes found in the input file.  May be None.
  70     manual_indexes: Optional datastore_index.IndexDefinitions instance
  71       containing indexes for which we should not generate output.  May be None.
  72
  73   Returns:
  74     A dict where each key is a tuple (kind, ancestor, properties) and the
  75       corresponding value is a count of the number of times that query has been
  76       issued. The dict contains no entries for keys that appear in manual_keys.
  77       In the tuple, "properties" is itself a tuple of tuples, where each
  78       contained tuple is (name, direction), with "name" being a string and
  79       "direction" being datastore_index.ASCENDING or .DESCENDING.
  80   """
  81
  82
  83
  84
  85
  86   all_keys = datastore_index.IndexDefinitionsToKeys(all_indexes)
  87   manual_keys = datastore_index.IndexDefinitionsToKeys(manual_indexes)
  88
  89
  90   indexes = dict((key, 0) for key in all_keys - manual_keys)
  91
  92
  93   for query, count in query_history.iteritems():
  94     required, kind, ancestor, props = (
  95         datastore_index.CompositeIndexForQuery(query))
  96     if required:
  97       props = datastore_index.GetRecommendedIndexProperties(props)
  98       key = (kind, ancestor, props)
  99       if key not in manual_keys:
 100         if key in indexes:
 101           indexes[key] += count
 102         else:
 103           indexes[key] = count
 104
 105   return indexes
 106
 107
 108 def GenerateIndexFromHistory(query_history,
 109                              all_indexes=None, manual_indexes=None):
 110   """Generate most of the text for index.yaml from the query history.
 111
 112   Args:
 113     query_history: Query history, a dict mapping datastore_pb.Query to a count
 114       of the number of times that query has been issued.
 115     all_indexes: Optional datastore_index.IndexDefinitions instance
 116       representing all the indexes found in the input file.  May be None.
 117     manual_indexes: Optional datastore_index.IndexDefinitions instance
 118       containing indexes for which we should not generate output.  May be None.
 119
 120   Returns:
 121     A string representation that can safely be appended to an existing
 122     index.yaml file. Returns the empty string if it would generate no output.
 123   """
 124   indexes = GenerateIndexDictFromHistory(
 125       query_history, all_indexes, manual_indexes)
 126
 127   if not indexes:
 128     return ''
 129
 130
 131
 132
 133   res = []
 134   for (kind, ancestor, props), _ in sorted(indexes.iteritems()):
 135
 136     res.append('')
 137     res.append(datastore_index.IndexYamlForQuery(kind, ancestor, props))
 138
 139   res.append('')
 140   return '\n'.join(res)
 141
 142
 143 class IndexYamlUpdater(object):
 144   """Helper class for updating index.yaml.
 145
 146   This class maintains some state about the query history and the
 147   index.yaml file in order to minimize the number of times index.yaml
 148   is actually overwritten.
 149   """
 150
 151
 152   index_yaml_is_manual = False
 153   index_yaml_mtime = None
 154   last_history_size = 0
 155
 156   def __init__(self, root_path):
 157     """Constructor.
 158
 159     Args:
 160       root_path: Path to the app's root directory.
 161     """
 162     self.root_path = root_path
 163
 164   def UpdateIndexConfig(self):
 165     self.UpdateIndexYaml()
 166
 167   def UpdateIndexYaml(self, openfile=open):
 168     """Update index.yaml.
 169
 170     Args:
 171       openfile: Used for dependency injection.
 172
 173     We only ever write to index.yaml if either:
 174     - it doesn't exist yet; or
 175     - it contains an 'AUTOGENERATED' comment.
 176
 177     All indexes *before* the AUTOGENERATED comment will be written
 178     back unchanged.  All indexes *after* the AUTOGENERATED comment
 179     will be updated with the latest query counts (query counts are
 180     reset by --clear_datastore).  Indexes that aren't yet in the file
 181     will be appended to the AUTOGENERATED section.
 182
 183     We keep track of some data in order to avoid doing repetitive work:
 184     - if index.yaml is fully manual, we keep track of its mtime to
 185       avoid parsing it over and over;
 186     - we keep track of the number of keys in the history dict since
 187       the last time we updated index.yaml (or decided there was
 188       nothing to update).
 189     """
 190
 191
 192
 193
 194
 195     index_yaml_file = os.path.join(self.root_path, 'index.yaml')
 196
 197
 198     try:
 199       index_yaml_mtime = os.path.getmtime(index_yaml_file)
 200     except os.error:
 201       index_yaml_mtime = None
 202
 203
 204     index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime)
 205     self.index_yaml_mtime = index_yaml_mtime
 206
 207
 208     datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 209     query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength()
 210     history_changed = (query_ci_history_len != self.last_history_size)
 211     self.last_history_size = query_ci_history_len
 212
 213
 214     if not (index_yaml_changed or history_changed):
 215       logging.debug('No need to update index.yaml')
 216       return
 217
 218
 219     if self.index_yaml_is_manual and not index_yaml_changed:
 220       logging.debug('Will not update manual index.yaml')
 221       return
 222
 223
 224     if index_yaml_mtime is None:
 225       index_yaml_data = None
 226     else:
 227       try:
 228
 229
 230
 231         fh = openfile(index_yaml_file, 'rU')
 232       except IOError:
 233         index_yaml_data = None
 234       else:
 235         try:
 236           index_yaml_data = fh.read()
 237         finally:
 238           fh.close()
 239
 240
 241     self.index_yaml_is_manual = (index_yaml_data is not None and
 242                                  AUTO_MARKER not in index_yaml_data)
 243     if self.index_yaml_is_manual:
 244       logging.info('Detected manual index.yaml, will not update')
 245       return
 246
 247
 248
 249     if index_yaml_data is None:
 250       all_indexes = None
 251     else:
 252       try:
 253         all_indexes = datastore_index.ParseIndexDefinitions(index_yaml_data)
 254       except yaml_errors.EventListenerError, e:
 255
 256         logging.error('Error parsing %s:\n%s', index_yaml_file, e)
 257         return
 258       except Exception, err:
 259
 260         logging.error('Error parsing %s:\n%s.%s: %s', index_yaml_file,
 261                       err.__class__.__module__, err.__class__.__name__, err)
 262         return
 263
 264
 265     if index_yaml_data is None:
 266       manual_part, prev_automatic_part = 'indexes:\n', ''
 267       manual_indexes = None
 268     else:
 269       manual_part, prev_automatic_part = index_yaml_data.split(AUTO_MARKER, 1)
 270       if prev_automatic_part.startswith(AUTO_COMMENT):
 271         prev_automatic_part = prev_automatic_part[len(AUTO_COMMENT):]
 272
 273       try:
 274         manual_indexes = datastore_index.ParseIndexDefinitions(manual_part)
 275       except Exception, err:
 276         logging.error('Error parsing manual part of %s: %s',
 277                       index_yaml_file, err)
 278         return
 279
 280
 281     automatic_part = GenerateIndexFromHistory(datastore_stub.QueryHistory(),
 282                                               all_indexes, manual_indexes)
 283
 284
 285
 286     if (index_yaml_mtime is None and automatic_part == '' or
 287         automatic_part == prev_automatic_part):
 288       logging.debug('No need to update index.yaml')
 289       return
 290
 291
 292     try:
 293       fh = openfile(index_yaml_file, 'w')
 294     except IOError, err:
 295       logging.error('Can\'t write index.yaml: %s', err)
 296       return
 297
 298
 299     try:
 300       logging.info('Updating %s', index_yaml_file)
 301       fh.write(manual_part)
 302       fh.write(AUTO_MARKER)
 303       fh.write(AUTO_COMMENT)
 304       fh.write(automatic_part)
 305     finally:
 306       fh.close()
 307
 308
 309     try:
 310       self.index_yaml_mtime = os.path.getmtime(index_yaml_file)
 311     except os.error, err:
 312       logging.error('Can\'t stat index.yaml we just wrote: %s', err)
 313       self.index_yaml_mtime = None
 314
 315
 316 class DatastoreIndexesAutoXmlUpdater(object):
 317   """Helper class for updating datastore-indexes-auto.xml.
 318
 319   This class maintains some state about the query history and the
 320   datastore-indexes.xml and datastore-indexes-auto.xml files in order to
 321   minimize the number of times datastore-indexes-auto.xml is rewritten.
 322   """
 323
 324
 325
 326   auto_generated = True
 327   datastore_indexes_xml = None
 328   datastore_indexes_xml_mtime = None
 329   datastore_indexes_auto_xml_mtime = None
 330   last_history_size = 0
 331
 332   def __init__(self, root_path):
 333     self.root_path = root_path
 334
 335   def UpdateIndexConfig(self):
 336     self.UpdateDatastoreIndexesAutoXml()
 337
 338   def UpdateDatastoreIndexesAutoXml(self, openfile=open):
 339     """Update datastore-indexes-auto.xml if appropriate."""
 340
 341
 342
 343
 344     datastore_indexes_xml_file = os.path.join(
 345         self.root_path, 'WEB-INF', 'datastore-indexes.xml')
 346     try:
 347       datastore_indexes_xml_mtime = os.path.getmtime(datastore_indexes_xml_file)
 348     except os.error:
 349       datastore_indexes_xml_mtime = None
 350     if datastore_indexes_xml_mtime != self.datastore_indexes_xml_mtime:
 351       self.datastore_indexes_xml_mtime = datastore_indexes_xml_mtime
 352       if self.datastore_indexes_xml_mtime:
 353         with openfile(datastore_indexes_xml_file) as f:
 354           self.datastore_indexes_xml = f.read()
 355           self.auto_generated = datastore_index_xml.IsAutoGenerated(
 356               self.datastore_indexes_xml)
 357       else:
 358         self.auto_generated = True
 359         self.datastore_indexes_xml = None
 360
 361     if not self.auto_generated:
 362       logging.debug('Detected <datastore-indexes autoGenerated="false">,'
 363                     ' will not update datastore-indexes-auto.xml')
 364       return
 365
 366
 367     datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 368     query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength()
 369     history_changed = (query_ci_history_len != self.last_history_size)
 370     self.last_history_size = query_ci_history_len
 371     if not history_changed:
 372       logging.debug('No need to update datastore-indexes-auto.xml')
 373       return
 374
 375     datastore_indexes_auto_xml_file = os.path.join(
 376         self.root_path, 'WEB-INF', 'appengine-generated',
 377         'datastore-indexes-auto.xml')
 378     try:
 379       with open(datastore_indexes_auto_xml_file) as f:
 380         datastore_indexes_auto_xml = f.read()
 381     except IOError, err:
 382       datastore_indexes_auto_xml = None
 383
 384     if self.datastore_indexes_xml:
 385       try:
 386         manual_index_definitions = (
 387             datastore_index_xml.IndexesXmlToIndexDefinitions(
 388                 self.datastore_indexes_xml))
 389       except validation.ValidationError, e:
 390         logging.error('Error parsing %s: %s',
 391                       datastore_indexes_xml_file, e)
 392         return
 393     else:
 394       manual_index_definitions = datastore_index.IndexDefinitions(indexes=[])
 395
 396     if datastore_indexes_auto_xml:
 397       try:
 398         prev_auto_index_definitions = (
 399             datastore_index_xml.IndexesXmlToIndexDefinitions(
 400                 datastore_indexes_auto_xml))
 401       except validation.ValidationError, e:
 402         logging.error('Error parsing %s: %s',
 403                       datastore_indexes_auto_xml_file, e)
 404         return
 405     else:
 406       prev_auto_index_definitions = datastore_index.IndexDefinitions(indexes=[])
 407
 408     all_index_definitions = datastore_index.IndexDefinitions(
 409         indexes=(manual_index_definitions.indexes +
 410                  prev_auto_index_definitions.indexes))
 411     query_history = datastore_stub.QueryHistory()
 412     auto_index_dict = GenerateIndexDictFromHistory(
 413         query_history, all_index_definitions, manual_index_definitions)
 414     auto_indexes, counts = self._IndexesFromIndexDict(auto_index_dict)
 415     auto_index_definitions = datastore_index.IndexDefinitions(
 416         indexes=auto_indexes)
 417     if auto_index_definitions == prev_auto_index_definitions:
 418       return
 419
 420     try:
 421       appengine_generated = os.path.dirname(datastore_indexes_auto_xml_file)
 422       if not os.path.exists(appengine_generated):
 423         os.mkdir(appengine_generated)
 424       with open(datastore_indexes_auto_xml_file, 'w') as f:
 425         f.write(self._IndexXmlFromIndexes(auto_indexes, counts))
 426     except os.error, err:
 427       logging.error(
 428           'Could not update %s: %s', datastore_indexes_auto_xml_file, err)
 429
 430   def _IndexesFromIndexDict(self, index_dict):
 431     """Convert a query dictionary into the corresponding required indexes.
 432
 433     Args:
 434       index_dict: Query history, a dict mapping datastore_pb.Query to a count
 435         of the number of times that query has been issued.
 436
 437     Returns:
 438       a tuple (indexes, counts) where indexes and counts are lists of the same
 439       size, with each entry in indexes being a datastore_index.Index and each
 440       entry in indexes being the count of the number of times the corresponding
 441       query appeared in the history.
 442     """
 443     indexes = []
 444     counts = []
 445     for (kind, ancestor, props), count in sorted(index_dict.iteritems()):
 446       properties = []
 447       for name, direction_code in props:
 448         direction = (
 449             'asc' if direction_code == datastore_index.ASCENDING else 'desc')
 450         properties.append(
 451             datastore_index.Property(name=name, direction=direction))
 452
 453       indexes.append(datastore_index.Index(
 454           kind=kind, ancestor=bool(ancestor), properties=properties))
 455       counts.append(count)
 456
 457     return indexes, counts
 458
 459   def _IndexXmlFromIndexes(self, indexes, counts):
 460     """Create <datastore-indexes> XML for the given indexes and query counts.
 461
 462     Args:
 463       indexes: a list of datastore_index.Index objects that are the required
 464         indexes.
 465       counts: a list of integers that are the corresponding counts.
 466
 467     Returns:
 468       the corresponding XML, with root node <datastore-indexes>.
 469     """
 470     lines = ['<datastore-indexes>']
 471     for index, count in zip(indexes, counts):
 472       lines.append('  <!-- Used %d time%s in query history -->'
 473                    % (count, 's' if count != 1 else ''))
 474       lines.append('  <datastore-index kind="%s" ancestor="%s">'
 475                    % (index.kind, 'true' if index.ancestor else 'false'))
 476       for prop in index.properties:
 477         lines.append('    <property name="%s" direction="%s" />'
 478                      % (prop.name, prop.direction))
 479       lines.append('  </datastore-index>')
 480     lines.append('</datastore-indexes>')
 481     return '\n'.join(lines) + '\n'