python/google/appengine/datastore/datastore_index.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Primitives for dealing with datastore indexes.
  22
  23 Example index.yaml file:
  24 ------------------------
  25
  26 indexes:
  27
  28 - kind: Cat
  29   ancestor: no
  30   properties:
  31   - name: name
  32   - name: age
  33     direction: desc
  34
  35 - kind: Cat
  36   properties:
  37   - name: name
  38     direction: ascending
  39   - name: whiskers
  40     direction: descending
  41
  42 - kind: Store
  43   ancestor: yes
  44   properties:
  45   - name: business
  46     direction: asc
  47   - name: owner
  48     direction: asc
  49
  50 - kind: Mountain
  51   properties:
  52   - name: location
  53     mode: geospatial
  54 """
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65 import google
  66 import yaml
  67
  68 import copy
  69 import itertools
  70
  71 from google.appengine.datastore import entity_pb
  72
  73 from google.appengine.api import appinfo
  74 from google.appengine.api import datastore_types
  75 from google.appengine.api import validation
  76 from google.appengine.api import yaml_errors
  77 from google.appengine.api import yaml_object
  78 from google.appengine.datastore import datastore_pb
  79
  80
  81 class Property(validation.Validated):
  82   """Representation for an individual property of an index.
  83
  84   This class must be kept in sync with
  85   java/com/google/apphosting/utils/config/IndexYamlReader.java.
  86
  87   Attributes:
  88     name: Name of attribute to sort by.
  89     direction: Direction of sort.
  90     mode: How the property is indexed. Either 'geospatial'
  91         or None (unspecified).
  92   """
  93
  94   ATTRIBUTES = {
  95       'name': validation.Type(str, convert=False),
  96       'direction': validation.Options(('asc', ('ascending',)),
  97                                       ('desc', ('descending',)),
  98                                       default='asc'),
  99       'mode': validation.Optional(['geospatial']),
 100       }
 101
 102   def __init__(self, **attributes):
 103
 104     object.__setattr__(self, '_is_set', set([]))
 105     super(Property, self).__init__(**attributes)
 106
 107   def __setattr__(self, key, value):
 108     self._is_set.add(key)
 109     super(Property, self).__setattr__(key, value)
 110
 111   def IsAscending(self):
 112     return self.direction != 'desc'
 113
 114   def CheckInitialized(self):
 115     if 'direction' in self._is_set and self.mode == 'geospatial':
 116       raise validation.ValidationError('Direction on a geospatial-mode '
 117                                        'property is not allowed.')
 118
 119
 120 def _PropertyPresenter(dumper, prop):
 121   """A PyYaml presenter for Property.
 122
 123   It differs from the default by not outputting 'mode: null' and direction when
 124   mode is specified. This is done in order to ensure backwards compatibility.
 125
 126   Args:
 127     dumper: the Dumper object provided by PyYaml.
 128     prop: the Property object to serialize.
 129
 130   Returns:
 131     A PyYaml object mapping.
 132   """
 133
 134
 135   prop_copy = copy.copy(prop)
 136
 137   del prop_copy._is_set
 138
 139
 140   if prop.mode is not None:
 141     del prop_copy.direction
 142   else:
 143     del prop_copy.mode
 144
 145   return dumper.represent_object(prop_copy)
 146
 147 yaml.add_representer(Property, _PropertyPresenter)
 148
 149
 150 class Index(validation.Validated):
 151   """Individual index definition.
 152
 153   Order of the properties determines a given indexes sort priority.
 154
 155   Attributes:
 156     kind: Datastore kind that index belongs to.
 157     ancestors: Include ancestors in index.
 158     properties: Properties to sort on.
 159   """
 160
 161   ATTRIBUTES = {
 162       'kind': validation.Type(str, convert=False),
 163       'ancestor': validation.Type(bool, convert=False, default=False),
 164       'properties': validation.Optional(validation.Repeated(Property)),
 165       }
 166
 167
 168 class IndexDefinitions(validation.Validated):
 169   """Top level for index definition file.
 170
 171   Attributes:
 172     indexes: List of Index definitions.
 173   """
 174
 175   ATTRIBUTES = {
 176       appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
 177       'indexes': validation.Optional(validation.Repeated(Index)),
 178       }
 179
 180
 181 def ParseIndexDefinitions(document, open_fn=None):
 182   """Parse an individual index definitions document from string or stream.
 183
 184   Args:
 185     document: Yaml document as a string or file-like stream.
 186     open_fn: Function for opening files. Unused.
 187
 188   Raises:
 189     EmptyConfigurationFile when the configuration file is empty.
 190     MultipleConfigurationFile when the configuration file contains more than
 191     one document.
 192
 193   Returns:
 194     Single parsed yaml file if one is defined, else None.
 195   """
 196   try:
 197     return yaml_object.BuildSingleObject(IndexDefinitions, document)
 198   except yaml_errors.EmptyConfigurationFile:
 199     return None
 200
 201
 202 def ParseMultipleIndexDefinitions(document):
 203   """Parse multiple index definitions documents from a string or stream.
 204
 205   Args:
 206     document: Yaml document as a string or file-like stream.
 207
 208   Returns:
 209     A list of datstore_index.IndexDefinitions objects, one for each document.
 210   """
 211   return yaml_object.BuildObjects(IndexDefinitions, document)
 212
 213
 214 def IndexDefinitionsToKeys(indexes):
 215   """Convert IndexDefinitions to set of keys.
 216
 217   Args:
 218     indexes: A datastore_index.IndexDefinitions instance, or None.
 219
 220   Returns:
 221     A set of keys constructed from the argument, each key being a
 222     tuple of the form (kind, ancestor, properties) where properties is
 223     a tuple of (name, direction) pairs, direction being ASCENDING or
 224     DESCENDING (the enums).
 225   """
 226   keyset = set()
 227   if indexes is not None:
 228     if indexes.indexes:
 229       for index in indexes.indexes:
 230         keyset.add(IndexToKey(index))
 231   return keyset
 232
 233
 234 def IndexToKey(index):
 235   """Convert Index to key.
 236
 237   Args:
 238     index: A datastore_index.Index instance (not None!).
 239
 240   Returns:
 241     A tuple of the form (kind, ancestor, properties) where properties
 242     is a tuple of (name, direction) pairs, direction being ASCENDING
 243     or DESCENDING (the enums).
 244   """
 245
 246
 247   props = []
 248   if index.properties is not None:
 249     for prop in index.properties:
 250       if prop.IsAscending():
 251         direction = ASCENDING
 252       else:
 253         direction = DESCENDING
 254       props.append((prop.name, direction))
 255   return index.kind, index.ancestor, tuple(props)
 256
 257
 258
 259
 260
 261
 262 ASCENDING = datastore_pb.Query_Order.ASCENDING
 263 DESCENDING = datastore_pb.Query_Order.DESCENDING
 264
 265
 266 EQUALITY_OPERATORS = set((datastore_pb.Query_Filter.EQUAL,
 267                           ))
 268 INEQUALITY_OPERATORS = set((datastore_pb.Query_Filter.LESS_THAN,
 269                             datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL,
 270                             datastore_pb.Query_Filter.GREATER_THAN,
 271                             datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL,
 272                             ))
 273 EXISTS_OPERATORS = set((datastore_pb.Query_Filter.EXISTS,
 274                         ))
 275
 276
 277 def Normalize(filters, orders, exists):
 278   """ Normalizes filter and order query components.
 279
 280   The resulting components have the same effect as the given components if used
 281   in a query.
 282
 283   Args:
 284     filters: the filters set on the query
 285     orders: the orders set on the query
 286     exists: the names of properties that require an exists filter if
 287       not already specified
 288
 289   Returns:
 290     (filter, orders) the reduced set of filters and orders
 291   """
 292
 293
 294   eq_properties = set()
 295   inequality_properties = set()
 296
 297
 298   for f in filters:
 299     if f.op() == datastore_pb.Query_Filter.IN and f.property_size() == 1:
 300       f.set_op(datastore_pb.Query_Filter.EQUAL)
 301     if f.op() in EQUALITY_OPERATORS:
 302       eq_properties.add(f.property(0).name())
 303     elif f.op() in INEQUALITY_OPERATORS:
 304       inequality_properties.add(f.property(0).name())
 305
 306   eq_properties -= inequality_properties
 307
 308
 309   remove_set = eq_properties.copy()
 310   new_orders = []
 311   for o in orders:
 312     if o.property() not in remove_set:
 313       remove_set.add(o.property())
 314       new_orders.append(o)
 315   orders = new_orders
 316
 317   remove_set.update(inequality_properties)
 318
 319
 320   new_filters = []
 321   for f in filters:
 322     if f.op() not in EXISTS_OPERATORS:
 323       new_filters.append(f)
 324       continue
 325     name = f.property(0).name()
 326     if name not in remove_set:
 327       remove_set.add(name)
 328       new_filters.append(f)
 329
 330
 331   for prop in exists:
 332     if prop not in remove_set:
 333       remove_set.add(prop)
 334       new_filter = datastore_pb.Query_Filter()
 335       new_filter.set_op(datastore_pb.Query_Filter.EXISTS)
 336       new_prop = new_filter.add_property()
 337       new_prop.set_name(prop)
 338       new_prop.set_multiple(False)
 339       new_prop.mutable_value()
 340       new_filters.append(new_filter)
 341
 342   filters = new_filters
 343
 344
 345
 346
 347   if datastore_types.KEY_SPECIAL_PROPERTY in eq_properties:
 348     orders = []
 349
 350
 351
 352   new_orders = []
 353   for o in orders:
 354     if o.property() == datastore_types.KEY_SPECIAL_PROPERTY:
 355       new_orders.append(o)
 356       break
 357     new_orders.append(o)
 358   orders = new_orders
 359
 360   return (filters, orders)
 361
 362
 363 def RemoveNativelySupportedComponents(filters, orders, exists):
 364   """ Removes query components that are natively supported by the datastore.
 365
 366   The resulting filters and orders should not be used in an actual query.
 367
 368   Args:
 369     filters: the filters set on the query
 370     orders: the orders set on the query
 371     exists: the names of properties that require an exists filter if
 372       not already specified
 373
 374   Returns:
 375     (filters, orders) the reduced set of filters and orders
 376   """
 377   (filters, orders) = Normalize(filters, orders, exists)
 378
 379   for f in filters:
 380     if f.op() in EXISTS_OPERATORS:
 381
 382
 383
 384       return (filters, orders)
 385
 386
 387
 388   has_key_desc_order = False
 389   if orders and orders[-1].property() == datastore_types.KEY_SPECIAL_PROPERTY:
 390     if orders[-1].direction() == ASCENDING:
 391       orders = orders[:-1]
 392     else:
 393       has_key_desc_order = True
 394
 395
 396
 397
 398
 399
 400   if not has_key_desc_order:
 401     for f in filters:
 402       if (f.op() in INEQUALITY_OPERATORS and
 403           f.property(0).name() != datastore_types.KEY_SPECIAL_PROPERTY):
 404         break
 405     else:
 406       filters = [f for f in filters
 407           if f.property(0).name() != datastore_types.KEY_SPECIAL_PROPERTY]
 408
 409   return (filters, orders)
 410
 411
 412 def CompositeIndexForQuery(query):
 413   """Return the composite index needed for a query.
 414
 415   A query is translated into a tuple, as follows:
 416
 417   - The first item is the kind string, or None if we're not filtering
 418     on kind (see below).
 419
 420   - The second item is a bool giving whether the query specifies an
 421     ancestor.
 422
 423   - After that come (property, ASCENDING) pairs for those Filter
 424     entries whose operator is EQUAL or IN.  Since the order of these
 425     doesn't matter, they are sorted by property name to normalize them
 426     in order to avoid duplicates.
 427
 428   - After that comes at most one (property, ASCENDING) pair for a
 429     Filter entry whose operator is on of the four inequalities.  There
 430     can be at most one of these.
 431
 432   - After that come all the (property, direction) pairs for the Order
 433     entries, in the order given in the query.  Exceptions:
 434       (a) if there is a Filter entry with an inequality operator that matches
 435           the first Order entry, the first order pair is omitted (or,
 436           equivalently, in this case the inequality pair is omitted).
 437       (b) if an Order entry corresponds to an equality filter, it is ignored
 438           (since there will only ever be one value returned).
 439       (c) if there is an equality filter on __key__ all orders are dropped
 440           (since there will be at most one result returned).
 441       (d) if there is an order on __key__ all further orders are dropped (since
 442           keys are unique).
 443       (e) orders on __key__ ASCENDING are dropped (since this is supported
 444           natively by the datastore).
 445
 446   - Finally, if there are Filter entries whose operator is EXISTS, and
 447     whose property names are not already listed, they are added, with
 448     the direction set to ASCENDING.
 449
 450   This algorithm should consume all Filter and Order entries.
 451
 452   Additional notes:
 453
 454   - The low-level implementation allows queries that don't specify a
 455     kind; but the Python API doesn't support this yet.
 456
 457   - If there's an inequality filter and one or more sort orders, the
 458     first sort order *must* match the inequality filter.
 459
 460   - The following indexes are always built in and should be suppressed:
 461     - query on kind only;
 462     - query on kind and one filter *or* one order;
 463     - query on ancestor only, without kind (not exposed in Python yet);
 464     - query on kind and equality filters only, no order (with or without
 465       ancestor).
 466
 467   - While the protocol buffer allows a Filter to contain multiple
 468     properties, we don't use this.  It is only needed for the IN operator
 469     but this is (currently) handled on the client side, so in practice
 470     each Filter is expected to have exactly one property.
 471
 472   Args:
 473     query: A datastore_pb.Query instance.
 474
 475   Returns:
 476     A tuple of the form (required, kind, ancestor, properties).
 477       required: boolean, whether the index is required;
 478       kind: the kind or None;
 479       ancestor: True if this is an ancestor query;
 480       properties: A tuple consisting of:
 481       - the prefix, represented by a set of property names
 482       - the postfix, represented by a tuple consisting of any number of:
 483         - Sets of property names: Indicates these properties can appear in any
 484           order with any direction.
 485         - Tuples of (property name, direction) tuples. Indicating the properties
 486           must appear in the exact order with the given direction. direction can
 487           be None if direction does not matter.
 488   """
 489   required = True
 490
 491
 492   kind = query.kind()
 493   ancestor = query.has_ancestor()
 494   filters = query.filter_list()
 495   orders = query.order_list()
 496
 497
 498
 499   for filter in filters:
 500     assert filter.op() != datastore_pb.Query_Filter.IN, 'Filter.op()==IN'
 501     nprops = len(filter.property_list())
 502     assert nprops == 1, 'Filter has %s properties, expected 1' % nprops
 503
 504   if not kind:
 505
 506
 507     required = False
 508
 509   exists = list(query.property_name_list())
 510   exists.extend(query.group_by_property_name_list())
 511
 512   filters, orders = RemoveNativelySupportedComponents(filters, orders, exists)
 513
 514
 515   eq_filters = [f for f in filters if f.op() in EQUALITY_OPERATORS]
 516   ineq_filters = [f for f in filters if f.op() in INEQUALITY_OPERATORS]
 517   exists_filters = [f for f in filters if f.op() in EXISTS_OPERATORS]
 518   assert (len(eq_filters) + len(ineq_filters) +
 519           len(exists_filters)) == len(filters), 'Not all filters used'
 520
 521   if (kind and not ineq_filters and not exists_filters and
 522       not orders):
 523
 524
 525
 526     names = set(f.property(0).name() for f in eq_filters)
 527     if not names.intersection(datastore_types._SPECIAL_PROPERTIES):
 528       required = False
 529
 530
 531
 532   ineq_property = None
 533   if ineq_filters:
 534     for filter in ineq_filters:
 535       if (filter.property(0).name() ==
 536           datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY):
 537         continue
 538       if not ineq_property:
 539         ineq_property = filter.property(0).name()
 540       else:
 541         assert filter.property(0).name() == ineq_property
 542
 543
 544
 545   group_by_props = set(query.group_by_property_name_list())
 546
 547
 548   prefix = frozenset(f.property(0).name() for f in eq_filters)
 549
 550   postfix_ordered = [(order.property(), order.direction()) for order in orders]
 551
 552
 553   postfix_group_by = frozenset(f.property(0).name() for f in exists_filters
 554                                if f.property(0).name() in group_by_props)
 555
 556   postfix_unordered = frozenset(f.property(0).name() for f in exists_filters
 557                                 if f.property(0).name() not in group_by_props)
 558
 559
 560   if ineq_property:
 561     if orders:
 562
 563
 564       assert ineq_property == orders[0].property()
 565     else:
 566       postfix_ordered.append((ineq_property, None))
 567
 568   property_count = (len(prefix) + len(postfix_ordered) + len(postfix_group_by)
 569                     + len(postfix_unordered))
 570   if kind and not ancestor and property_count <= 1:
 571
 572
 573     required = False
 574
 575
 576     if postfix_ordered:
 577       prop, dir = postfix_ordered[0]
 578       if prop == datastore_types.KEY_SPECIAL_PROPERTY and dir is DESCENDING:
 579         required = True
 580
 581
 582   props = prefix, (tuple(postfix_ordered), postfix_group_by, postfix_unordered)
 583   return required, kind, ancestor, props
 584
 585
 586 def GetRecommendedIndexProperties(properties):
 587   """Converts the properties returned by datastore_index.CompositeIndexForQuery
 588   into a recommended list of index properties and directions.
 589
 590   All unordered components are sorted and assigned an ASCENDING direction. All
 591   ordered components with out a direction are assigned an ASCEDNING direction.
 592
 593   Args:
 594     properties: See datastore_index.CompositeIndexForQuery
 595
 596   Returns:
 597     A tuple of (name, direction) tuples where:
 598         name: a property name
 599         direction: datastore_pb.Query_Order.ASCENDING or ...DESCENDING
 600   """
 601
 602   prefix, postfix = properties
 603   result = []
 604   for sub_list in itertools.chain((prefix,), postfix):
 605     if isinstance(sub_list, (frozenset, set)):
 606
 607       for prop in sorted(sub_list):
 608         result.append((prop, ASCENDING))
 609     else:
 610
 611
 612       for prop, dir in sub_list:
 613         result.append((prop, dir if dir is not None else ASCENDING))
 614
 615   return tuple(result)
 616
 617
 618 def _MatchPostfix(postfix_props, index_props):
 619   """Matches a postfix constraint with an existing index.
 620
 621   postfix_props constraints are specified through a list of:
 622   - sets of string: any order any direction;
 623   - list of tuples(string, direction): the given order, and, if specified, the
 624   given direction.
 625
 626   For example:
 627     [set('A', 'B'), [('C', None), ('D', ASC)]]
 628   matches:
 629     [('F', ASC), ('B', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 630   with a return value of [('F', ASC)], but does not match:
 631     [('F', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 632     [('B', ASC), ('F', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 633     [('F', ASC), ('B', ASC), ('A', DESC), ('C', DESC), ('D', DESC)]
 634
 635   Args:
 636     postfix_props: A tuple of sets and lists, as output by
 637         CompositeIndexForQuery. They should define the requirements for the
 638         postfix of the index.
 639     index_props: A list of tuples (property_name, property_direction), that
 640         define the index to try and match.
 641
 642   Returns:
 643     The list of tuples that define the prefix properties in the given index.
 644     None if the constraints could not be satisfied.
 645   """
 646
 647
 648   index_props_rev = reversed(index_props)
 649   for property_group in reversed(postfix_props):
 650     index_group_iter = itertools.islice(index_props_rev, len(property_group))
 651     if isinstance(property_group, (frozenset, set)):
 652
 653       index_group = set(prop for prop, _ in index_group_iter)
 654       if index_group != property_group:
 655         return None
 656     else:
 657
 658       index_group = list(index_group_iter)
 659       if len(index_group) != len(property_group):
 660         return None
 661       for (index_prop, index_dir), (prop, direction) in itertools.izip(
 662           index_group, reversed(property_group)):
 663         if index_prop != prop or (direction and index_dir != direction):
 664           return None
 665   remaining = list(index_props_rev)
 666   remaining.reverse()
 667   return remaining
 668
 669
 670 def MinimalCompositeIndexForQuery(query, index_defs):
 671   """Computes the minimal composite index for this query.
 672
 673   Unlike datastore_index.CompositeIndexForQuery, this function takes into
 674   account indexes that already exist in the system.
 675
 676   Args:
 677     query: the datastore_pb.Query to compute suggestions for
 678     index_defs: a list of datastore_index.Index objects that already exist.
 679
 680   Returns:
 681     None if no index is needed, otherwise the minimal index in the form
 682   (is_most_efficient, kind, ancestor, properties). Where is_most_efficient is a
 683   boolean denoting if the suggested index is the most efficient (i.e. the one
 684   returned by datastore_index.CompositeIndexForQuery). kind and ancestor
 685   are the same variables returned by datastore_index.CompositeIndexForQuery.
 686   properties is a tuple consisting of the prefix and postfix properties
 687   returend by datastore_index.CompositeIndexForQuery.
 688   """
 689
 690   required, kind, ancestor, (prefix, postfix) = CompositeIndexForQuery(query)
 691
 692   if not required:
 693     return None
 694
 695
 696   remaining_dict = {}
 697
 698   for definition in index_defs:
 699     if (kind != definition.kind or
 700
 701         (not ancestor and definition.ancestor)):
 702       continue
 703
 704     _, _, index_props = IndexToKey(definition)
 705
 706     index_prefix = _MatchPostfix(postfix, index_props)
 707
 708     if index_prefix is None:
 709
 710       continue
 711
 712     remaining_index_props = set([prop for prop, _ in index_prefix])
 713
 714     if remaining_index_props - prefix:
 715
 716       continue
 717
 718
 719     index_postfix = tuple(index_props[len(index_prefix):])
 720     remaining = remaining_dict.get(index_postfix)
 721     if remaining is None:
 722       remaining = prefix.copy(), ancestor
 723
 724
 725     props_remaining, ancestor_remaining = remaining
 726     props_remaining -= remaining_index_props
 727     if definition.ancestor:
 728       ancestor_remaining = False
 729
 730     if not (props_remaining or ancestor_remaining):
 731       return None
 732
 733     if (props_remaining, ancestor_remaining) == remaining:
 734       continue
 735
 736
 737     remaining_dict[index_postfix] = (props_remaining, ancestor_remaining)
 738
 739   if not remaining_dict:
 740     return (True, kind, ancestor, (prefix, postfix))
 741
 742   def calc_cost(minimal_props, minimal_ancestor):
 743     result = len(minimal_props)
 744     if minimal_ancestor:
 745       result += 2
 746     return result
 747
 748
 749   minimal_postfix, remaining = remaining_dict.popitem()
 750   minimal_props, minimal_ancestor = remaining
 751   minimal_cost = calc_cost(minimal_props, minimal_ancestor)
 752   for index_postfix, (props_remaining, ancestor_remaining) in (
 753       remaining_dict.iteritems()):
 754     cost = calc_cost(props_remaining, ancestor_remaining)
 755     if cost < minimal_cost:
 756       minimal_cost = cost
 757       minimal_postfix = index_postfix
 758       minimal_props = props_remaining
 759       minimal_ancestor = ancestor_remaining
 760
 761
 762   props = frozenset(minimal_props), (minimal_postfix, frozenset(), frozenset())
 763   return False, kind, minimal_ancestor, props
 764
 765
 766 def IndexYamlForQuery(kind, ancestor, props):
 767   """Return the composite index definition YAML needed for a query.
 768
 769   Given a query, the arguments for this method can be computed with:
 770     _, kind, ancestor, props = datastore_index.CompositeIndexForQuery(query)
 771     props = datastore_index.GetRecommendedIndexProperties(props)
 772
 773   Args:
 774     kind: the kind or None
 775     ancestor: True if this is an ancestor query, False otherwise
 776     props: tuples of the form (name, direction) where:
 777         name - a property name;
 778         direction - datastore_pb.Query_Order.ASCENDING or ...DESCENDING;
 779
 780   Returns:
 781     A string with the YAML for the composite index needed by the query.
 782   """
 783
 784
 785   serialized_yaml = []
 786   serialized_yaml.append('- kind: %s' % kind)
 787   if ancestor:
 788     serialized_yaml.append('  ancestor: yes')
 789   if props:
 790     serialized_yaml.append('  properties:')
 791     for name, direction in props:
 792       serialized_yaml.append('  - name: %s' % name)
 793       if direction == DESCENDING:
 794         serialized_yaml.append('    direction: desc')
 795   return '\n'.join(serialized_yaml)
 796
 797
 798 def IndexXmlForQuery(kind, ancestor, props):
 799   """Return the composite index definition XML needed for a query.
 800
 801   Given a query, the arguments for this method can be computed with:
 802     _, kind, ancestor, props = datastore_index.CompositeIndexForQuery(query)
 803     props = datastore_index.GetRecommendedIndexProperties(props)
 804
 805   Args:
 806     kind: the kind or None
 807     ancestor: True if this is an ancestor query, False otherwise
 808     props: tuples of the form (name, direction) where:
 809         name - a property name;
 810         direction - datastore_pb.Query_Order.ASCENDING or ...DESCENDING;
 811
 812   Returns:
 813     A string with the XML for the composite index needed by the query.
 814   """
 815
 816
 817   serialized_xml = []
 818   serialized_xml.append('<datastore-index kind="%s" ancestor="%s">'
 819                         % (kind, 'true' if ancestor else 'false'))
 820   for name, direction in props:
 821     serialized_xml.append('  <property name="%s" direction="%s" />'
 822                           % (name, 'asc' if direction == ASCENDING else 'desc'))
 823   serialized_xml.append('</datastore-index>')
 824   return '\n'.join(serialized_xml)
 825
 826
 827 def IndexDefinitionToProto(app_id, index_definition):
 828   """Transform individual Index definition to protocol buffer.
 829
 830   Args:
 831     app_id: Application id for new protocol buffer CompositeIndex.
 832     index_definition: datastore_index.Index object to transform.
 833
 834   Returns:
 835     New entity_pb.CompositeIndex with default values set and index
 836     information filled in.
 837   """
 838   proto = entity_pb.CompositeIndex()
 839
 840   proto.set_app_id(app_id)
 841   proto.set_id(0)
 842   proto.set_state(entity_pb.CompositeIndex.WRITE_ONLY)
 843
 844   definition_proto = proto.mutable_definition()
 845   definition_proto.set_entity_type(index_definition.kind)
 846   definition_proto.set_ancestor(index_definition.ancestor)
 847
 848   if index_definition.properties is not None:
 849     for prop in index_definition.properties:
 850       prop_proto = definition_proto.add_property()
 851       prop_proto.set_name(prop.name)
 852
 853       if prop.mode == 'geospatial':
 854         prop_proto.set_mode(entity_pb.Index_Property.GEOSPATIAL)
 855       elif prop.IsAscending():
 856         prop_proto.set_direction(entity_pb.Index_Property.ASCENDING)
 857       else:
 858         prop_proto.set_direction(entity_pb.Index_Property.DESCENDING)
 859
 860   return proto
 861
 862
 863 def IndexDefinitionsToProtos(app_id, index_definitions):
 864   """Transform multiple index definitions to composite index records
 865
 866   Args:
 867     app_id: Application id for new protocol buffer CompositeIndex.
 868     index_definition: A list of datastore_index.Index objects to transform.
 869
 870   Returns:
 871     A list of tranformed entity_pb.Compositeindex entities with default values
 872     set and index information filled in.
 873   """
 874   return [IndexDefinitionToProto(app_id, index)
 875           for index in index_definitions]
 876
 877
 878 def ProtoToIndexDefinition(proto):
 879   """Transform individual index protocol buffer to index definition.
 880
 881   Args:
 882     proto: An instance of entity_pb.CompositeIndex to transform.
 883
 884   Returns:
 885     A new instance of datastore_index.Index.
 886   """
 887   properties = []
 888   proto_index = proto.definition()
 889   for prop_proto in proto_index.property_list():
 890     prop_definition = Property(name=prop_proto.name())
 891
 892     if prop_proto.mode() == entity_pb.Index_Property.GEOSPATIAL:
 893       prop_definition.mode = 'geospatial'
 894     elif prop_proto.direction() == entity_pb.Index_Property.DESCENDING:
 895       prop_definition.direction = 'desc'
 896     elif prop_proto.direction() == entity_pb.Index_Property.ASCENDING:
 897       prop_definition.direction = 'asc'
 898
 899     properties.append(prop_definition)
 900
 901   index = Index(kind=proto_index.entity_type(), properties=properties)
 902   if proto_index.ancestor():
 903     index.ancestor = True
 904   return index
 905
 906
 907 def ProtosToIndexDefinitions(protos):
 908   """Transform multiple index protocol buffers to index definitions.
 909
 910   Args:
 911     A list of entity_pb.Index records.
 912   """
 913   return [ProtoToIndexDefinition(definition) for definition in protos]