python/google/appengine/datastore/datastore_index.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19 """Primitives for dealing with datastore indexes.
  20
  21 Example index.yaml file:
  22 ------------------------
  23
  24 indexes:
  25
  26 - kind: Cat
  27   ancestor: no
  28   properties:
  29   - name: name
  30   - name: age
  31     direction: desc
  32
  33 - kind: Cat
  34   properties:
  35   - name: name
  36     direction: ascending
  37   - name: whiskers
  38     direction: descending
  39
  40 - kind: Store
  41   ancestor: yes
  42   properties:
  43   - name: business
  44     direction: asc
  45   - name: owner
  46     direction: asc
  47
  48 - kind: Mountain
  49   properties:
  50   - name: name
  51     mode: segment
  52   - name: location
  53     mode: geospatial
  54 """
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65 import google
  66 import yaml
  67
  68 import copy
  69 import itertools
  70
  71 from google.appengine.datastore import entity_pb
  72
  73 from google.appengine.api import appinfo
  74 from google.appengine.api import datastore_types
  75 from google.appengine.api import validation
  76 from google.appengine.api import yaml_errors
  77 from google.appengine.api import yaml_object
  78 from google.appengine.datastore import datastore_pb
  79
  80
  81 class Property(validation.Validated):
  82   """Representation for an individual property of an index.
  83
  84   This class must be kept in sync with
  85   java/com/google/apphosting/utils/config/IndexYamlReader.java.
  86
  87   Attributes:
  88     name: Name of attribute to sort by.
  89     direction: Direction of sort.
  90     mode: How the property is indexed. Either 'geospatial', 'segment'
  91         or None (unspecified).
  92   """
  93
  94   ATTRIBUTES = {
  95       'name': validation.Type(str, convert=False),
  96       'direction': validation.Options(('asc', ('ascending',)),
  97                                       ('desc', ('descending',)),
  98                                       default='asc'),
  99       'mode': validation.Optional(['geospatial', 'segment']),
 100   }
 101
 102   def __init__(self, **attributes):
 103
 104     object.__setattr__(self, '_is_set', set([]))
 105     super(Property, self).__init__(**attributes)
 106
 107   def __setattr__(self, key, value):
 108     self._is_set.add(key)
 109     super(Property, self).__setattr__(key, value)
 110
 111   def IsAscending(self):
 112     return self.direction != 'desc'
 113
 114   def CheckInitialized(self):
 115     if ('direction' in self._is_set and
 116         self.mode in ['geospatial', 'segment']):
 117       raise validation.ValidationError('Direction on a %s-mode '
 118                                        'property is not allowed.' % self.mode)
 119
 120
 121 def _PropertyPresenter(dumper, prop):
 122   """A PyYaml presenter for Property.
 123
 124   It differs from the default by not outputting 'mode: null' and direction when
 125   mode is specified. This is done in order to ensure backwards compatibility.
 126
 127   Args:
 128     dumper: the Dumper object provided by PyYaml.
 129     prop: the Property object to serialize.
 130
 131   Returns:
 132     A PyYaml object mapping.
 133   """
 134
 135
 136   prop_copy = copy.copy(prop)
 137
 138   del prop_copy._is_set
 139
 140
 141   if prop.mode is not None:
 142     del prop_copy.direction
 143   else:
 144     del prop_copy.mode
 145
 146   return dumper.represent_object(prop_copy)
 147
 148 yaml.add_representer(Property, _PropertyPresenter)
 149
 150
 151 class Index(validation.Validated):
 152   """Individual index definition.
 153
 154   Order of the properties determines a given indexes sort priority.
 155
 156   Attributes:
 157     kind: Datastore kind that index belongs to.
 158     ancestors: Include ancestors in index.
 159     properties: Properties to sort on.
 160   """
 161
 162   ATTRIBUTES = {
 163       'kind': validation.Type(str, convert=False),
 164       'ancestor': validation.Type(bool, convert=False, default=False),
 165       'properties': validation.Optional(validation.Repeated(Property)),
 166   }
 167
 168
 169 class IndexDefinitions(validation.Validated):
 170   """Top level for index definition file.
 171
 172   Attributes:
 173     indexes: List of Index definitions.
 174   """
 175
 176   ATTRIBUTES = {
 177       appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
 178       'indexes': validation.Optional(validation.Repeated(Index)),
 179   }
 180
 181
 182 def ParseIndexDefinitions(document, open_fn=None):
 183   """Parse an individual index definitions document from string or stream.
 184
 185   Args:
 186     document: Yaml document as a string or file-like stream.
 187     open_fn: Function for opening files. Unused.
 188
 189   Raises:
 190     EmptyConfigurationFile when the configuration file is empty.
 191     MultipleConfigurationFile when the configuration file contains more than
 192     one document.
 193
 194   Returns:
 195     Single parsed yaml file if one is defined, else None.
 196   """
 197   try:
 198     return yaml_object.BuildSingleObject(IndexDefinitions, document)
 199   except yaml_errors.EmptyConfigurationFile:
 200     return None
 201
 202
 203 def ParseMultipleIndexDefinitions(document):
 204   """Parse multiple index definitions documents from a string or stream.
 205
 206   Args:
 207     document: Yaml document as a string or file-like stream.
 208
 209   Returns:
 210     A list of datstore_index.IndexDefinitions objects, one for each document.
 211   """
 212   return yaml_object.BuildObjects(IndexDefinitions, document)
 213
 214
 215 def IndexDefinitionsToKeys(indexes):
 216   """Convert IndexDefinitions to set of keys.
 217
 218   Args:
 219     indexes: A datastore_index.IndexDefinitions instance, or None.
 220
 221   Returns:
 222     A set of keys constructed from the argument, each key being a
 223     tuple of the form (kind, ancestor, properties) where properties is
 224     a tuple of (name, direction) pairs, direction being ASCENDING or
 225     DESCENDING (the enums).
 226   """
 227   keyset = set()
 228   if indexes is not None:
 229     if indexes.indexes:
 230       for index in indexes.indexes:
 231         keyset.add(IndexToKey(index))
 232   return keyset
 233
 234
 235 def IndexToKey(index):
 236   """Convert Index to key.
 237
 238   Args:
 239     index: A datastore_index.Index instance (not None!).
 240
 241   Returns:
 242     A tuple of the form (kind, ancestor, properties) where properties
 243     is a tuple of (name, direction) pairs, direction being ASCENDING
 244     or DESCENDING (the enums).
 245   """
 246
 247
 248   props = []
 249   if index.properties is not None:
 250     for prop in index.properties:
 251       if prop.IsAscending():
 252         direction = ASCENDING
 253       else:
 254         direction = DESCENDING
 255       props.append((prop.name, direction))
 256   return index.kind, index.ancestor, tuple(props)
 257
 258
 259
 260
 261
 262
 263 ASCENDING = datastore_pb.Query_Order.ASCENDING
 264 DESCENDING = datastore_pb.Query_Order.DESCENDING
 265
 266
 267 EQUALITY_OPERATORS = set([datastore_pb.Query_Filter.EQUAL])
 268 INEQUALITY_OPERATORS = set([datastore_pb.Query_Filter.LESS_THAN,
 269                             datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL,
 270                             datastore_pb.Query_Filter.GREATER_THAN,
 271                             datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL])
 272 EXISTS_OPERATORS = set([datastore_pb.Query_Filter.EXISTS])
 273
 274
 275 def Normalize(filters, orders, exists):
 276   """ Normalizes filter and order query components.
 277
 278   The resulting components have the same effect as the given components if used
 279   in a query.
 280
 281   Args:
 282     filters: the filters set on the query
 283     orders: the orders set on the query
 284     exists: the names of properties that require an exists filter if
 285       not already specified
 286
 287   Returns:
 288     (filter, orders) the reduced set of filters and orders
 289   """
 290
 291
 292   eq_properties = set()
 293   inequality_properties = set()
 294
 295
 296   for f in filters:
 297     if f.op() == datastore_pb.Query_Filter.IN and f.property_size() == 1:
 298       f.set_op(datastore_pb.Query_Filter.EQUAL)
 299     if f.op() in EQUALITY_OPERATORS:
 300       eq_properties.add(f.property(0).name())
 301     elif f.op() in INEQUALITY_OPERATORS:
 302       inequality_properties.add(f.property(0).name())
 303
 304   eq_properties -= inequality_properties
 305
 306
 307   remove_set = eq_properties.copy()
 308   new_orders = []
 309   for o in orders:
 310     if o.property() not in remove_set:
 311       remove_set.add(o.property())
 312       new_orders.append(o)
 313   orders = new_orders
 314
 315   remove_set.update(inequality_properties)
 316
 317
 318   new_filters = []
 319   for f in filters:
 320     if f.op() not in EXISTS_OPERATORS:
 321       new_filters.append(f)
 322       continue
 323     name = f.property(0).name()
 324     if name not in remove_set:
 325       remove_set.add(name)
 326       new_filters.append(f)
 327
 328
 329   for prop in exists:
 330     if prop not in remove_set:
 331       remove_set.add(prop)
 332       new_filter = datastore_pb.Query_Filter()
 333       new_filter.set_op(datastore_pb.Query_Filter.EXISTS)
 334       new_prop = new_filter.add_property()
 335       new_prop.set_name(prop)
 336       new_prop.set_multiple(False)
 337       new_prop.mutable_value()
 338       new_filters.append(new_filter)
 339
 340   filters = new_filters
 341
 342
 343
 344
 345   if datastore_types.KEY_SPECIAL_PROPERTY in eq_properties:
 346     orders = []
 347
 348
 349
 350   new_orders = []
 351   for o in orders:
 352     if o.property() == datastore_types.KEY_SPECIAL_PROPERTY:
 353       new_orders.append(o)
 354       break
 355     new_orders.append(o)
 356   orders = new_orders
 357
 358   return (filters, orders)
 359
 360
 361 def RemoveNativelySupportedComponents(filters, orders, exists):
 362   """ Removes query components that are natively supported by the datastore.
 363
 364   The resulting filters and orders should not be used in an actual query.
 365
 366   Args:
 367     filters: the filters set on the query
 368     orders: the orders set on the query
 369     exists: the names of properties that require an exists filter if
 370       not already specified
 371
 372   Returns:
 373     (filters, orders) the reduced set of filters and orders
 374   """
 375   (filters, orders) = Normalize(filters, orders, exists)
 376
 377   for f in filters:
 378     if f.op() in EXISTS_OPERATORS:
 379
 380
 381
 382       return (filters, orders)
 383
 384
 385   has_key_desc_order = False
 386   if orders and orders[-1].property() == datastore_types.KEY_SPECIAL_PROPERTY:
 387     if orders[-1].direction() == ASCENDING:
 388       orders = orders[:-1]
 389     else:
 390       has_key_desc_order = True
 391
 392
 393
 394
 395
 396
 397   if not has_key_desc_order:
 398     for f in filters:
 399       if (f.op() in INEQUALITY_OPERATORS and
 400           f.property(0).name() != datastore_types.KEY_SPECIAL_PROPERTY):
 401         break
 402     else:
 403       filters = [
 404           f for f in filters
 405           if f.property(0).name() != datastore_types.KEY_SPECIAL_PROPERTY]
 406
 407   return (filters, orders)
 408
 409
 410 def CompositeIndexForQuery(query):
 411   """Return the composite index needed for a query.
 412
 413   A query is translated into a tuple, as follows:
 414
 415   - The first item is the kind string, or None if we're not filtering
 416     on kind (see below).
 417
 418   - The second item is a bool giving whether the query specifies an
 419     ancestor.
 420
 421   - After that come (property, ASCENDING) pairs for those Filter
 422     entries whose operator is EQUAL or IN.  Since the order of these
 423     doesn't matter, they are sorted by property name to normalize them
 424     in order to avoid duplicates.
 425
 426   - After that comes at most one (property, ASCENDING) pair for a
 427     Filter entry whose operator is on of the four inequalities.  There
 428     can be at most one of these.
 429
 430   - After that come all the (property, direction) pairs for the Order
 431     entries, in the order given in the query.  Exceptions:
 432       (a) if there is a Filter entry with an inequality operator that matches
 433           the first Order entry, the first order pair is omitted (or,
 434           equivalently, in this case the inequality pair is omitted).
 435       (b) if an Order entry corresponds to an equality filter, it is ignored
 436           (since there will only ever be one value returned).
 437       (c) if there is an equality filter on __key__ all orders are dropped
 438           (since there will be at most one result returned).
 439       (d) if there is an order on __key__ all further orders are dropped (since
 440           keys are unique).
 441       (e) orders on __key__ ASCENDING are dropped (since this is supported
 442           natively by the datastore).
 443
 444   - Finally, if there are Filter entries whose operator is EXISTS, and
 445     whose property names are not already listed, they are added, with
 446     the direction set to ASCENDING.
 447
 448   This algorithm should consume all Filter and Order entries.
 449
 450   Additional notes:
 451
 452   - The low-level implementation allows queries that don't specify a
 453     kind; but the Python API doesn't support this yet.
 454
 455   - If there's an inequality filter and one or more sort orders, the
 456     first sort order *must* match the inequality filter.
 457
 458   - The following indexes are always built in and should be suppressed:
 459     - query on kind only;
 460     - query on kind and one filter *or* one order;
 461     - query on ancestor only, without kind (not exposed in Python yet);
 462     - query on kind and equality filters only, no order (with or without
 463       ancestor).
 464
 465   - While the protocol buffer allows a Filter to contain multiple
 466     properties, we don't use this.  It is only needed for the IN operator
 467     but this is (currently) handled on the client side, so in practice
 468     each Filter is expected to have exactly one property.
 469
 470   Args:
 471     query: A datastore_pb.Query instance.
 472
 473   Returns:
 474     A tuple of the form (required, kind, ancestor, properties).
 475       required: boolean, whether the index is required;
 476       kind: the kind or None;
 477       ancestor: True if this is an ancestor query;
 478       properties: A tuple consisting of:
 479       - the prefix, represented by a set of property names
 480       - the postfix, represented by a tuple consisting of any number of:
 481         - Sets of property names: Indicates these properties can appear in any
 482           order with any direction.
 483         - Tuples of (property name, direction) tuples. Indicating the properties
 484           must appear in the exact order with the given direction. direction can
 485           be None if direction does not matter.
 486   """
 487   required = True
 488
 489
 490   kind = query.kind()
 491   ancestor = query.has_ancestor()
 492   filters = query.filter_list()
 493   orders = query.order_list()
 494
 495
 496
 497   for filter in filters:
 498     assert filter.op() != datastore_pb.Query_Filter.IN, 'Filter.op()==IN'
 499     nprops = len(filter.property_list())
 500     assert nprops == 1, 'Filter has %s properties, expected 1' % nprops
 501
 502   if not kind:
 503
 504
 505     required = False
 506
 507   exists = list(query.property_name_list())
 508   exists.extend(query.group_by_property_name_list())
 509
 510   filters, orders = RemoveNativelySupportedComponents(filters, orders, exists)
 511
 512
 513   eq_filters = [f for f in filters if f.op() in EQUALITY_OPERATORS]
 514   ineq_filters = [f for f in filters if f.op() in INEQUALITY_OPERATORS]
 515   exists_filters = [f for f in filters if f.op() in EXISTS_OPERATORS]
 516   assert (len(eq_filters) + len(ineq_filters) +
 517           len(exists_filters)) == len(filters), 'Not all filters used'
 518
 519   if (kind and not ineq_filters and not exists_filters and
 520       not orders):
 521
 522
 523
 524     names = set(f.property(0).name() for f in eq_filters)
 525     if not names.intersection(datastore_types._SPECIAL_PROPERTIES):
 526       required = False
 527
 528
 529
 530   ineq_property = None
 531   if ineq_filters:
 532     for filter in ineq_filters:
 533       if (filter.property(0).name() ==
 534           datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY):
 535         continue
 536       if not ineq_property:
 537         ineq_property = filter.property(0).name()
 538       else:
 539         assert filter.property(0).name() == ineq_property
 540
 541
 542
 543   group_by_props = set(query.group_by_property_name_list())
 544
 545
 546   prefix = frozenset(f.property(0).name() for f in eq_filters)
 547
 548   postfix_ordered = [(order.property(), order.direction()) for order in orders]
 549
 550
 551   postfix_group_by = frozenset(f.property(0).name() for f in exists_filters
 552                                if f.property(0).name() in group_by_props)
 553
 554   postfix_unordered = frozenset(f.property(0).name() for f in exists_filters
 555                                 if f.property(0).name() not in group_by_props)
 556
 557
 558   if ineq_property:
 559     if orders:
 560
 561
 562       assert ineq_property == orders[0].property()
 563     else:
 564       postfix_ordered.append((ineq_property, None))
 565
 566   property_count = (len(prefix) + len(postfix_ordered) + len(postfix_group_by)
 567                     + len(postfix_unordered))
 568   if kind and not ancestor and property_count <= 1:
 569
 570
 571     required = False
 572
 573
 574     if postfix_ordered:
 575       prop, dir = postfix_ordered[0]
 576       if prop == datastore_types.KEY_SPECIAL_PROPERTY and dir is DESCENDING:
 577         required = True
 578
 579
 580   props = prefix, (tuple(postfix_ordered), postfix_group_by, postfix_unordered)
 581   return required, kind, ancestor, props
 582
 583
 584 def GetRecommendedIndexProperties(properties):
 585   """Converts the properties returned by datastore_index.CompositeIndexForQuery
 586   into a recommended list of index properties and directions.
 587
 588   All unordered components are sorted and assigned an ASCENDING direction. All
 589   ordered components with out a direction are assigned an ASCEDNING direction.
 590
 591   Args:
 592     properties: See datastore_index.CompositeIndexForQuery
 593
 594   Returns:
 595     A tuple of (name, direction) tuples where:
 596         name: a property name
 597         direction: datastore_pb.Query_Order.ASCENDING or ...DESCENDING
 598   """
 599
 600   prefix, postfix = properties
 601   result = []
 602   for sub_list in itertools.chain((prefix,), postfix):
 603     if isinstance(sub_list, (frozenset, set)):
 604
 605       for prop in sorted(sub_list):
 606         result.append((prop, ASCENDING))
 607     else:
 608
 609
 610       for prop, dir in sub_list:
 611         result.append((prop, dir if dir is not None else ASCENDING))
 612
 613   return tuple(result)
 614
 615
 616 def _MatchPostfix(postfix_props, index_props):
 617   """Matches a postfix constraint with an existing index.
 618
 619   postfix_props constraints are specified through a list of:
 620   - sets of string: any order any direction;
 621   - list of tuples(string, direction): the given order, and, if specified, the
 622   given direction.
 623
 624   For example:
 625     [set('A', 'B'), [('C', None), ('D', ASC)]]
 626   matches:
 627     [('F', ASC), ('B', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 628   with a return value of [('F', ASC)], but does not match:
 629     [('F', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 630     [('B', ASC), ('F', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 631     [('F', ASC), ('B', ASC), ('A', DESC), ('C', DESC), ('D', DESC)]
 632
 633   Args:
 634     postfix_props: A tuple of sets and lists, as output by
 635         CompositeIndexForQuery. They should define the requirements for the
 636         postfix of the index.
 637     index_props: A list of tuples (property_name, property_direction), that
 638         define the index to try and match.
 639
 640   Returns:
 641     The list of tuples that define the prefix properties in the given index.
 642     None if the constraints could not be satisfied.
 643   """
 644
 645
 646   index_props_rev = reversed(index_props)
 647   for property_group in reversed(postfix_props):
 648     index_group_iter = itertools.islice(index_props_rev, len(property_group))
 649     if isinstance(property_group, (frozenset, set)):
 650
 651       index_group = set(prop for prop, _ in index_group_iter)
 652       if index_group != property_group:
 653         return None
 654     else:
 655
 656       index_group = list(index_group_iter)
 657       if len(index_group) != len(property_group):
 658         return None
 659       for (index_prop, index_dir), (prop, direction) in itertools.izip(
 660           index_group, reversed(property_group)):
 661         if index_prop != prop or (direction and index_dir != direction):
 662           return None
 663   remaining = list(index_props_rev)
 664   remaining.reverse()
 665   return remaining
 666
 667
 668 def MinimalCompositeIndexForQuery(query, index_defs):
 669   """Computes the minimal composite index for this query.
 670
 671   Unlike datastore_index.CompositeIndexForQuery, this function takes into
 672   account indexes that already exist in the system.
 673
 674   Args:
 675     query: the datastore_pb.Query to compute suggestions for
 676     index_defs: a list of datastore_index.Index objects that already exist.
 677
 678   Returns:
 679     None if no index is needed, otherwise the minimal index in the form
 680   (is_most_efficient, kind, ancestor, properties). Where is_most_efficient is a
 681   boolean denoting if the suggested index is the most efficient (i.e. the one
 682   returned by datastore_index.CompositeIndexForQuery). kind and ancestor
 683   are the same variables returned by datastore_index.CompositeIndexForQuery.
 684   properties is a tuple consisting of the prefix and postfix properties
 685   returend by datastore_index.CompositeIndexForQuery.
 686   """
 687
 688   required, kind, ancestor, (prefix, postfix) = CompositeIndexForQuery(query)
 689
 690   if not required:
 691     return None
 692
 693
 694   remaining_dict = {}
 695
 696   for definition in index_defs:
 697     if (kind != definition.kind or
 698
 699         (not ancestor and definition.ancestor)):
 700       continue
 701
 702     _, _, index_props = IndexToKey(definition)
 703
 704     index_prefix = _MatchPostfix(postfix, index_props)
 705
 706     if index_prefix is None:
 707
 708       continue
 709
 710     remaining_index_props = set([prop for prop, _ in index_prefix])
 711
 712     if remaining_index_props - prefix:
 713
 714       continue
 715
 716
 717     index_postfix = tuple(index_props[len(index_prefix):])
 718     remaining = remaining_dict.get(index_postfix)
 719     if remaining is None:
 720       remaining = prefix.copy(), ancestor
 721
 722
 723     props_remaining, ancestor_remaining = remaining
 724     props_remaining -= remaining_index_props
 725     if definition.ancestor:
 726       ancestor_remaining = False
 727
 728     if not (props_remaining or ancestor_remaining):
 729       return None
 730
 731     if (props_remaining, ancestor_remaining) == remaining:
 732       continue
 733
 734
 735     remaining_dict[index_postfix] = (props_remaining, ancestor_remaining)
 736
 737   if not remaining_dict:
 738     return (True, kind, ancestor, (prefix, postfix))
 739
 740   def calc_cost(minimal_props, minimal_ancestor):
 741     result = len(minimal_props)
 742     if minimal_ancestor:
 743       result += 2
 744     return result
 745
 746
 747   minimal_postfix, remaining = remaining_dict.popitem()
 748   minimal_props, minimal_ancestor = remaining
 749   minimal_cost = calc_cost(minimal_props, minimal_ancestor)
 750   for index_postfix, (props_remaining, ancestor_remaining) in (
 751       remaining_dict.iteritems()):
 752     cost = calc_cost(props_remaining, ancestor_remaining)
 753     if cost < minimal_cost:
 754       minimal_cost = cost
 755       minimal_postfix = index_postfix
 756       minimal_props = props_remaining
 757       minimal_ancestor = ancestor_remaining
 758
 759
 760   props = frozenset(minimal_props), (minimal_postfix, frozenset(), frozenset())
 761   return False, kind, minimal_ancestor, props
 762
 763
 764
 765
 766
 767
 768 def IndexYamlForQuery(kind, ancestor, props):
 769   """Return the composite index definition YAML needed for a query.
 770
 771   Given a query, the arguments for this method can be computed with:
 772     _, kind, ancestor, props = datastore_index.CompositeIndexForQuery(query)
 773     props = datastore_index.GetRecommendedIndexProperties(props)
 774
 775   Args:
 776     kind: the kind or None
 777     ancestor: True if this is an ancestor query, False otherwise
 778     props: tuples of the form (name, direction) where:
 779         name - a property name;
 780         direction - datastore_pb.Query_Order.ASCENDING or ...DESCENDING;
 781
 782   Returns:
 783     A string with the YAML for the composite index needed by the query.
 784   """
 785
 786
 787   serialized_yaml = []
 788   serialized_yaml.append('- kind: %s' % kind)
 789   if ancestor:
 790     serialized_yaml.append('  ancestor: yes')
 791   if props:
 792     serialized_yaml.append('  properties:')
 793     for name, direction in props:
 794       serialized_yaml.append('  - name: %s' % name)
 795       if direction == DESCENDING:
 796         serialized_yaml.append('    direction: desc')
 797   return '\n'.join(serialized_yaml)
 798
 799
 800 def IndexXmlForQuery(kind, ancestor, props):
 801   """Return the composite index definition XML needed for a query.
 802
 803   Given a query, the arguments for this method can be computed with:
 804     _, kind, ancestor, props = datastore_index.CompositeIndexForQuery(query)
 805     props = datastore_index.GetRecommendedIndexProperties(props)
 806
 807   Args:
 808     kind: the kind or None
 809     ancestor: True if this is an ancestor query, False otherwise
 810     props: tuples of the form (name, direction) where:
 811         name - a property name;
 812         direction - datastore_pb.Query_Order.ASCENDING or ...DESCENDING;
 813
 814   Returns:
 815     A string with the XML for the composite index needed by the query.
 816   """
 817
 818
 819   serialized_xml = []
 820   serialized_xml.append('  <datastore-index kind="%s" ancestor="%s">'
 821                         % (kind, 'true' if ancestor else 'false'))
 822   for name, direction in props:
 823     serialized_xml.append('    <property name="%s" direction="%s" />'
 824                           % (name, 'asc' if direction == ASCENDING else 'desc'))
 825   serialized_xml.append('  </datastore-index>')
 826   return '\n'.join(serialized_xml)
 827
 828
 829 def IndexDefinitionToProto(app_id, index_definition):
 830   """Transform individual Index definition to protocol buffer.
 831
 832   Args:
 833     app_id: Application id for new protocol buffer CompositeIndex.
 834     index_definition: datastore_index.Index object to transform.
 835
 836   Returns:
 837     New entity_pb.CompositeIndex with default values set and index
 838     information filled in.
 839   """
 840   proto = entity_pb.CompositeIndex()
 841
 842   proto.set_app_id(app_id)
 843   proto.set_id(0)
 844   proto.set_state(entity_pb.CompositeIndex.WRITE_ONLY)
 845
 846   definition_proto = proto.mutable_definition()
 847   definition_proto.set_entity_type(index_definition.kind)
 848   definition_proto.set_ancestor(index_definition.ancestor)
 849
 850   if index_definition.properties is not None:
 851     for prop in index_definition.properties:
 852       prop_proto = definition_proto.add_property()
 853       prop_proto.set_name(prop.name)
 854
 855       if prop.mode == 'geospatial':
 856         prop_proto.set_mode(entity_pb.Index_Property.GEOSPATIAL)
 857       elif prop.mode == 'segment':
 858         prop_proto.set_mode(entity_pb.Index_Property.SEGMENT)
 859       elif prop.IsAscending():
 860         prop_proto.set_direction(entity_pb.Index_Property.ASCENDING)
 861       else:
 862         prop_proto.set_direction(entity_pb.Index_Property.DESCENDING)
 863
 864   return proto
 865
 866
 867 def IndexDefinitionsToProtos(app_id, index_definitions):
 868   """Transform multiple index definitions to composite index records
 869
 870   Args:
 871     app_id: Application id for new protocol buffer CompositeIndex.
 872     index_definition: A list of datastore_index.Index objects to transform.
 873
 874   Returns:
 875     A list of tranformed entity_pb.Compositeindex entities with default values
 876     set and index information filled in.
 877   """
 878   return [IndexDefinitionToProto(app_id, index)
 879           for index in index_definitions]
 880
 881
 882 def ProtoToIndexDefinition(proto):
 883   """Transform individual index protocol buffer to index definition.
 884
 885   Args:
 886     proto: An instance of entity_pb.CompositeIndex to transform.
 887
 888   Returns:
 889     A new instance of datastore_index.Index.
 890   """
 891   properties = []
 892   proto_index = proto.definition()
 893   for prop_proto in proto_index.property_list():
 894     prop_definition = Property(name=prop_proto.name())
 895
 896     if prop_proto.mode() == entity_pb.Index_Property.GEOSPATIAL:
 897       prop_definition.mode = 'geospatial'
 898     elif prop_proto.mode() == entity_pb.Index_Property.SEGMENT:
 899       prop_definition.mode = 'segment'
 900     elif prop_proto.direction() == entity_pb.Index_Property.DESCENDING:
 901       prop_definition.direction = 'desc'
 902     elif prop_proto.direction() == entity_pb.Index_Property.ASCENDING:
 903       prop_definition.direction = 'asc'
 904
 905     properties.append(prop_definition)
 906
 907   index = Index(kind=proto_index.entity_type(), properties=properties)
 908   if proto_index.ancestor():
 909     index.ancestor = True
 910   return index
 911
 912
 913 def ProtosToIndexDefinitions(protos):
 914   """Transform multiple index protocol buffers to index definitions.
 915
 916   Args:
 917     A list of entity_pb.Index records.
 918   """
 919   return [ProtoToIndexDefinition(definition) for definition in protos]