python/google/appengine/datastore/datastore_index.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19 """Primitives for dealing with datastore indexes.
  20
  21 Example index.yaml file:
  22 ------------------------
  23
  24 indexes:
  25
  26 - kind: Cat
  27   ancestor: no
  28   properties:
  29   - name: name
  30   - name: age
  31     direction: desc
  32
  33 - kind: Cat
  34   properties:
  35   - name: name
  36     direction: ascending
  37   - name: whiskers
  38     direction: descending
  39
  40 - kind: Store
  41   ancestor: yes
  42   properties:
  43   - name: business
  44     direction: asc
  45   - name: owner
  46     direction: asc
  47
  48 - kind: Mountain
  49   properties:
  50   - name: name
  51     mode: segment
  52   - name: location
  53     mode: geospatial
  54 """
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67 import google
  68 import yaml
  69
  70 import copy
  71 import itertools
  72
  73 from google.appengine.datastore import entity_pb
  74
  75 from google.appengine.api import appinfo
  76 from google.appengine.api import datastore_types
  77 from google.appengine.api import validation
  78 from google.appengine.api import yaml_errors
  79 from google.appengine.api import yaml_object
  80 from google.appengine.datastore import datastore_pb
  81
  82
  83 class Property(validation.Validated):
  84   """Representation for an individual property of an index.
  85
  86   This class must be kept in sync with
  87   java/com/google/apphosting/utils/config/IndexYamlReader.java.
  88
  89   Attributes:
  90     name: Name of attribute to sort by.
  91     direction: Direction of sort.
  92     mode: How the property is indexed. Either 'geospatial', 'segment'
  93         or None (unspecified).
  94   """
  95
  96   ATTRIBUTES = {
  97       'name': validation.Type(str, convert=False),
  98       'direction': validation.Options(('asc', ('ascending',)),
  99                                       ('desc', ('descending',)),
 100                                       default='asc'),
 101       'mode': validation.Optional(['geospatial', 'segment']),
 102   }
 103
 104   def __init__(self, **attributes):
 105
 106     object.__setattr__(self, '_is_set', set([]))
 107     super(Property, self).__init__(**attributes)
 108
 109   def __setattr__(self, key, value):
 110     self._is_set.add(key)
 111     super(Property, self).__setattr__(key, value)
 112
 113   def IsAscending(self):
 114     return self.direction != 'desc'
 115
 116   def CheckInitialized(self):
 117     if ('direction' in self._is_set and
 118         self.mode in ['geospatial', 'segment']):
 119       raise validation.ValidationError('Direction on a %s-mode '
 120                                        'property is not allowed.' % self.mode)
 121
 122
 123 def _PropertyPresenter(dumper, prop):
 124   """A PyYaml presenter for Property.
 125
 126   It differs from the default by not outputting 'mode: null' and direction when
 127   mode is specified. This is done in order to ensure backwards compatibility.
 128
 129   Args:
 130     dumper: the Dumper object provided by PyYaml.
 131     prop: the Property object to serialize.
 132
 133   Returns:
 134     A PyYaml object mapping.
 135   """
 136
 137
 138   prop_copy = copy.copy(prop)
 139
 140   del prop_copy._is_set
 141
 142
 143   if prop.mode is not None:
 144     del prop_copy.direction
 145   else:
 146     del prop_copy.mode
 147
 148   return dumper.represent_object(prop_copy)
 149
 150 yaml.add_representer(Property, _PropertyPresenter)
 151
 152
 153 class Index(validation.Validated):
 154   """Individual index definition.
 155
 156   Order of the properties determines a given indexes sort priority.
 157
 158   Attributes:
 159     kind: Datastore kind that index belongs to.
 160     ancestors: Include ancestors in index.
 161     properties: Properties to sort on.
 162   """
 163
 164   ATTRIBUTES = {
 165       'kind': validation.Type(str, convert=False),
 166       'ancestor': validation.Type(bool, convert=False, default=False),
 167       'properties': validation.Optional(validation.Repeated(Property)),
 168   }
 169
 170
 171 class IndexDefinitions(validation.Validated):
 172   """Top level for index definition file.
 173
 174   Attributes:
 175     indexes: List of Index definitions.
 176   """
 177
 178   ATTRIBUTES = {
 179       appinfo.APPLICATION: validation.Optional(appinfo.APPLICATION_RE_STRING),
 180       'indexes': validation.Optional(validation.Repeated(Index)),
 181   }
 182
 183
 184 def ParseIndexDefinitions(document, open_fn=None):
 185   """Parse an individual index definitions document from string or stream.
 186
 187   Args:
 188     document: Yaml document as a string or file-like stream.
 189     open_fn: Function for opening files. Unused.
 190
 191   Raises:
 192     EmptyConfigurationFile when the configuration file is empty.
 193     MultipleConfigurationFile when the configuration file contains more than
 194     one document.
 195
 196   Returns:
 197     Single parsed yaml file if one is defined, else None.
 198   """
 199   try:
 200     return yaml_object.BuildSingleObject(IndexDefinitions, document)
 201   except yaml_errors.EmptyConfigurationFile:
 202     return None
 203
 204
 205 def ParseMultipleIndexDefinitions(document):
 206   """Parse multiple index definitions documents from a string or stream.
 207
 208   Args:
 209     document: Yaml document as a string or file-like stream.
 210
 211   Returns:
 212     A list of datstore_index.IndexDefinitions objects, one for each document.
 213   """
 214   return yaml_object.BuildObjects(IndexDefinitions, document)
 215
 216
 217 def IndexDefinitionsToKeys(indexes):
 218   """Convert IndexDefinitions to set of keys.
 219
 220   Args:
 221     indexes: A datastore_index.IndexDefinitions instance, or None.
 222
 223   Returns:
 224     A set of keys constructed from the argument, each key being a
 225     tuple of the form (kind, ancestor, properties) where properties is
 226     a tuple of (name, direction) pairs, direction being ASCENDING or
 227     DESCENDING (the enums).
 228   """
 229   keyset = set()
 230   if indexes is not None:
 231     if indexes.indexes:
 232       for index in indexes.indexes:
 233         keyset.add(IndexToKey(index))
 234   return keyset
 235
 236
 237 def IndexToKey(index):
 238   """Convert Index to key.
 239
 240   Args:
 241     index: A datastore_index.Index instance (not None!).
 242
 243   Returns:
 244     A tuple of the form (kind, ancestor, properties) where properties
 245     is a tuple of (name, direction) pairs, direction being ASCENDING
 246     or DESCENDING (the enums).
 247   """
 248
 249
 250   props = []
 251   if index.properties is not None:
 252     for prop in index.properties:
 253       if prop.IsAscending():
 254         direction = ASCENDING
 255       else:
 256         direction = DESCENDING
 257       props.append((prop.name, direction))
 258   return index.kind, index.ancestor, tuple(props)
 259
 260
 261
 262
 263
 264
 265 ASCENDING = datastore_pb.Query_Order.ASCENDING
 266 DESCENDING = datastore_pb.Query_Order.DESCENDING
 267
 268
 269 EQUALITY_OPERATORS = set([datastore_pb.Query_Filter.EQUAL])
 270 INEQUALITY_OPERATORS = set([datastore_pb.Query_Filter.LESS_THAN,
 271                             datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL,
 272                             datastore_pb.Query_Filter.GREATER_THAN,
 273                             datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL])
 274 EXISTS_OPERATORS = set([datastore_pb.Query_Filter.EXISTS])
 275
 276
 277 def Normalize(filters, orders, exists):
 278   """ Normalizes filter and order query components.
 279
 280   The resulting components have the same effect as the given components if used
 281   in a query.
 282
 283   Args:
 284     filters: the filters set on the query
 285     orders: the orders set on the query
 286     exists: the names of properties that require an exists filter if
 287       not already specified
 288
 289   Returns:
 290     (filter, orders) the reduced set of filters and orders
 291   """
 292
 293
 294   eq_properties = set()
 295   inequality_properties = set()
 296
 297
 298   for f in filters:
 299     if f.op() == datastore_pb.Query_Filter.IN and f.property_size() == 1:
 300       f.set_op(datastore_pb.Query_Filter.EQUAL)
 301     if f.op() in EQUALITY_OPERATORS:
 302       eq_properties.add(f.property(0).name())
 303     elif f.op() in INEQUALITY_OPERATORS:
 304       inequality_properties.add(f.property(0).name())
 305
 306   eq_properties -= inequality_properties
 307
 308
 309   remove_set = eq_properties.copy()
 310   new_orders = []
 311   for o in orders:
 312     if o.property() not in remove_set:
 313       remove_set.add(o.property())
 314       new_orders.append(o)
 315   orders = new_orders
 316
 317   remove_set.update(inequality_properties)
 318
 319
 320   new_filters = []
 321   for f in filters:
 322     if f.op() not in EXISTS_OPERATORS:
 323       new_filters.append(f)
 324       continue
 325     name = f.property(0).name()
 326     if name not in remove_set:
 327       remove_set.add(name)
 328       new_filters.append(f)
 329
 330
 331   for prop in exists:
 332     if prop not in remove_set:
 333       remove_set.add(prop)
 334       new_filter = datastore_pb.Query_Filter()
 335       new_filter.set_op(datastore_pb.Query_Filter.EXISTS)
 336       new_prop = new_filter.add_property()
 337       new_prop.set_name(prop)
 338       new_prop.set_multiple(False)
 339       new_prop.mutable_value()
 340       new_filters.append(new_filter)
 341
 342   filters = new_filters
 343
 344
 345
 346
 347   if datastore_types.KEY_SPECIAL_PROPERTY in eq_properties:
 348     orders = []
 349
 350
 351
 352   new_orders = []
 353   for o in orders:
 354     if o.property() == datastore_types.KEY_SPECIAL_PROPERTY:
 355       new_orders.append(o)
 356       break
 357     new_orders.append(o)
 358   orders = new_orders
 359
 360   return (filters, orders)
 361
 362
 363 def RemoveNativelySupportedComponents(filters, orders, exists):
 364   """ Removes query components that are natively supported by the datastore.
 365
 366   The resulting filters and orders should not be used in an actual query.
 367
 368   Args:
 369     filters: the filters set on the query
 370     orders: the orders set on the query
 371     exists: the names of properties that require an exists filter if
 372       not already specified
 373
 374   Returns:
 375     (filters, orders) the reduced set of filters and orders
 376   """
 377   (filters, orders) = Normalize(filters, orders, exists)
 378
 379   for f in filters:
 380     if f.op() in EXISTS_OPERATORS:
 381
 382
 383
 384       return (filters, orders)
 385
 386
 387   has_key_desc_order = False
 388   if orders and orders[-1].property() == datastore_types.KEY_SPECIAL_PROPERTY:
 389     if orders[-1].direction() == ASCENDING:
 390       orders = orders[:-1]
 391     else:
 392       has_key_desc_order = True
 393
 394
 395
 396
 397
 398
 399   if not has_key_desc_order:
 400     for f in filters:
 401       if (f.op() in INEQUALITY_OPERATORS and
 402           f.property(0).name() != datastore_types.KEY_SPECIAL_PROPERTY):
 403         break
 404     else:
 405       filters = [
 406           f for f in filters
 407           if f.property(0).name() != datastore_types.KEY_SPECIAL_PROPERTY]
 408
 409   return (filters, orders)
 410
 411
 412 def CompositeIndexForQuery(query):
 413   """Return the composite index needed for a query.
 414
 415   A query is translated into a tuple, as follows:
 416
 417   - The first item is the kind string, or None if we're not filtering
 418     on kind (see below).
 419
 420   - The second item is a bool giving whether the query specifies an
 421     ancestor.
 422
 423   - After that come (property, ASCENDING) pairs for those Filter
 424     entries whose operator is EQUAL or IN.  Since the order of these
 425     doesn't matter, they are sorted by property name to normalize them
 426     in order to avoid duplicates.
 427
 428   - After that comes at most one (property, ASCENDING) pair for a
 429     Filter entry whose operator is on of the four inequalities.  There
 430     can be at most one of these.
 431
 432   - After that come all the (property, direction) pairs for the Order
 433     entries, in the order given in the query.  Exceptions:
 434       (a) if there is a Filter entry with an inequality operator that matches
 435           the first Order entry, the first order pair is omitted (or,
 436           equivalently, in this case the inequality pair is omitted).
 437       (b) if an Order entry corresponds to an equality filter, it is ignored
 438           (since there will only ever be one value returned).
 439       (c) if there is an equality filter on __key__ all orders are dropped
 440           (since there will be at most one result returned).
 441       (d) if there is an order on __key__ all further orders are dropped (since
 442           keys are unique).
 443       (e) orders on __key__ ASCENDING are dropped (since this is supported
 444           natively by the datastore).
 445
 446   - Finally, if there are Filter entries whose operator is EXISTS, and
 447     whose property names are not already listed, they are added, with
 448     the direction set to ASCENDING.
 449
 450   This algorithm should consume all Filter and Order entries.
 451
 452   Additional notes:
 453
 454   - The low-level implementation allows queries that don't specify a
 455     kind; but the Python API doesn't support this yet.
 456
 457   - If there's an inequality filter and one or more sort orders, the
 458     first sort order *must* match the inequality filter.
 459
 460   - The following indexes are always built in and should be suppressed:
 461     - query on kind only;
 462     - query on kind and one filter *or* one order;
 463     - query on ancestor only, without kind (not exposed in Python yet);
 464     - query on kind and equality filters only, no order (with or without
 465       ancestor).
 466
 467   - While the protocol buffer allows a Filter to contain multiple
 468     properties, we don't use this.  It is only needed for the IN operator
 469     but this is (currently) handled on the client side, so in practice
 470     each Filter is expected to have exactly one property.
 471
 472   Args:
 473     query: A datastore_pb.Query instance.
 474
 475   Returns:
 476     A tuple of the form (required, kind, ancestor, properties).
 477       required: boolean, whether the index is required;
 478       kind: the kind or None;
 479       ancestor: True if this is an ancestor query;
 480       properties: A tuple consisting of:
 481       - the prefix, represented by a set of property names
 482       - the postfix, represented by a tuple consisting of any number of:
 483         - Sets of property names: Indicates these properties can appear in any
 484           order with any direction.
 485         - Tuples of (property name, direction) tuples. Indicating the properties
 486           must appear in the exact order with the given direction. direction can
 487           be None if direction does not matter.
 488   """
 489   required = True
 490
 491
 492   kind = query.kind()
 493   ancestor = query.has_ancestor()
 494   filters = query.filter_list()
 495   orders = query.order_list()
 496
 497
 498
 499   for filter in filters:
 500     assert filter.op() != datastore_pb.Query_Filter.IN, 'Filter.op()==IN'
 501     nprops = len(filter.property_list())
 502     assert nprops == 1, 'Filter has %s properties, expected 1' % nprops
 503
 504   if not kind:
 505
 506
 507     required = False
 508
 509   exists = list(query.property_name_list())
 510   exists.extend(query.group_by_property_name_list())
 511
 512   filters, orders = RemoveNativelySupportedComponents(filters, orders, exists)
 513
 514
 515   eq_filters = [f for f in filters if f.op() in EQUALITY_OPERATORS]
 516   ineq_filters = [f for f in filters if f.op() in INEQUALITY_OPERATORS]
 517   exists_filters = [f for f in filters if f.op() in EXISTS_OPERATORS]
 518   assert (len(eq_filters) + len(ineq_filters) +
 519           len(exists_filters)) == len(filters), 'Not all filters used'
 520
 521   if (kind and not ineq_filters and not exists_filters and
 522       not orders):
 523
 524
 525
 526     names = set(f.property(0).name() for f in eq_filters)
 527     if not names.intersection(datastore_types._SPECIAL_PROPERTIES):
 528       required = False
 529
 530
 531
 532   ineq_property = None
 533   if ineq_filters:
 534     for filter in ineq_filters:
 535       if (filter.property(0).name() ==
 536           datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY):
 537         continue
 538       if not ineq_property:
 539         ineq_property = filter.property(0).name()
 540       else:
 541         assert filter.property(0).name() == ineq_property
 542
 543
 544
 545   group_by_props = set(query.group_by_property_name_list())
 546
 547
 548   prefix = frozenset(f.property(0).name() for f in eq_filters)
 549
 550   postfix_ordered = [(order.property(), order.direction()) for order in orders]
 551
 552
 553   postfix_group_by = frozenset(f.property(0).name() for f in exists_filters
 554                                if f.property(0).name() in group_by_props)
 555
 556   postfix_unordered = frozenset(f.property(0).name() for f in exists_filters
 557                                 if f.property(0).name() not in group_by_props)
 558
 559
 560   if ineq_property:
 561     if orders:
 562
 563
 564       assert ineq_property == orders[0].property()
 565     else:
 566       postfix_ordered.append((ineq_property, None))
 567
 568   property_count = (len(prefix) + len(postfix_ordered) + len(postfix_group_by)
 569                     + len(postfix_unordered))
 570   if kind and not ancestor and property_count <= 1:
 571
 572
 573     required = False
 574
 575
 576     if postfix_ordered:
 577       prop, dir = postfix_ordered[0]
 578       if prop == datastore_types.KEY_SPECIAL_PROPERTY and dir is DESCENDING:
 579         required = True
 580
 581
 582   props = prefix, (tuple(postfix_ordered), postfix_group_by, postfix_unordered)
 583   return required, kind, ancestor, props
 584
 585
 586 def GetRecommendedIndexProperties(properties):
 587   """Converts the properties returned by datastore_index.CompositeIndexForQuery
 588   into a recommended list of index properties and directions.
 589
 590   All unordered components are sorted and assigned an ASCENDING direction. All
 591   ordered components with out a direction are assigned an ASCEDNING direction.
 592
 593   Args:
 594     properties: See datastore_index.CompositeIndexForQuery
 595
 596   Returns:
 597     A tuple of (name, direction) tuples where:
 598         name: a property name
 599         direction: datastore_pb.Query_Order.ASCENDING or ...DESCENDING
 600   """
 601
 602   prefix, postfix = properties
 603   result = []
 604   for sub_list in itertools.chain((prefix,), postfix):
 605     if isinstance(sub_list, (frozenset, set)):
 606
 607       for prop in sorted(sub_list):
 608         result.append((prop, ASCENDING))
 609     else:
 610
 611
 612       for prop, dir in sub_list:
 613         result.append((prop, dir if dir is not None else ASCENDING))
 614
 615   return tuple(result)
 616
 617
 618 def _MatchPostfix(postfix_props, index_props):
 619   """Matches a postfix constraint with an existing index.
 620
 621   postfix_props constraints are specified through a list of:
 622   - sets of string: any order any direction;
 623   - list of tuples(string, direction): the given order, and, if specified, the
 624   given direction.
 625
 626   For example:
 627     [set('A', 'B'), [('C', None), ('D', ASC)]]
 628   matches:
 629     [('F', ASC), ('B', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 630   with a return value of [('F', ASC)], but does not match:
 631     [('F', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 632     [('B', ASC), ('F', ASC), ('A', DESC), ('C', DESC), ('D', ASC)]
 633     [('F', ASC), ('B', ASC), ('A', DESC), ('C', DESC), ('D', DESC)]
 634
 635   Args:
 636     postfix_props: A tuple of sets and lists, as output by
 637         CompositeIndexForQuery. They should define the requirements for the
 638         postfix of the index.
 639     index_props: A list of tuples (property_name, property_direction), that
 640         define the index to try and match.
 641
 642   Returns:
 643     The list of tuples that define the prefix properties in the given index.
 644     None if the constraints could not be satisfied.
 645   """
 646
 647
 648   index_props_rev = reversed(index_props)
 649   for property_group in reversed(postfix_props):
 650     index_group_iter = itertools.islice(index_props_rev, len(property_group))
 651     if isinstance(property_group, (frozenset, set)):
 652
 653       index_group = set(prop for prop, _ in index_group_iter)
 654       if index_group != property_group:
 655         return None
 656     else:
 657
 658       index_group = list(index_group_iter)
 659       if len(index_group) != len(property_group):
 660         return None
 661       for (index_prop, index_dir), (prop, direction) in itertools.izip(
 662           index_group, reversed(property_group)):
 663         if index_prop != prop or (direction and index_dir != direction):
 664           return None
 665   remaining = list(index_props_rev)
 666   remaining.reverse()
 667   return remaining
 668
 669
 670 def MinimalCompositeIndexForQuery(query, index_defs):
 671   """Computes the minimal composite index for this query.
 672
 673   Unlike datastore_index.CompositeIndexForQuery, this function takes into
 674   account indexes that already exist in the system.
 675
 676   Args:
 677     query: the datastore_pb.Query to compute suggestions for
 678     index_defs: a list of datastore_index.Index objects that already exist.
 679
 680   Returns:
 681     None if no index is needed, otherwise the minimal index in the form
 682   (is_most_efficient, kind, ancestor, properties). Where is_most_efficient is a
 683   boolean denoting if the suggested index is the most efficient (i.e. the one
 684   returned by datastore_index.CompositeIndexForQuery). kind and ancestor
 685   are the same variables returned by datastore_index.CompositeIndexForQuery.
 686   properties is a tuple consisting of the prefix and postfix properties
 687   returend by datastore_index.CompositeIndexForQuery.
 688   """
 689
 690   required, kind, ancestor, (prefix, postfix) = CompositeIndexForQuery(query)
 691
 692   if not required:
 693     return None
 694
 695
 696   remaining_dict = {}
 697
 698   for definition in index_defs:
 699     if (kind != definition.kind or
 700
 701         (not ancestor and definition.ancestor)):
 702       continue
 703
 704     _, _, index_props = IndexToKey(definition)
 705
 706     index_prefix = _MatchPostfix(postfix, index_props)
 707
 708     if index_prefix is None:
 709
 710       continue
 711
 712     remaining_index_props = set([prop for prop, _ in index_prefix])
 713
 714     if remaining_index_props - prefix:
 715
 716       continue
 717
 718
 719     index_postfix = tuple(index_props[len(index_prefix):])
 720     remaining = remaining_dict.get(index_postfix)
 721     if remaining is None:
 722       remaining = prefix.copy(), ancestor
 723
 724
 725     props_remaining, ancestor_remaining = remaining
 726     props_remaining -= remaining_index_props
 727     if definition.ancestor:
 728       ancestor_remaining = False
 729
 730     if not (props_remaining or ancestor_remaining):
 731       return None
 732
 733     if (props_remaining, ancestor_remaining) == remaining:
 734       continue
 735
 736
 737     remaining_dict[index_postfix] = (props_remaining, ancestor_remaining)
 738
 739   if not remaining_dict:
 740     return (True, kind, ancestor, (prefix, postfix))
 741
 742   def calc_cost(minimal_props, minimal_ancestor):
 743     result = len(minimal_props)
 744     if minimal_ancestor:
 745       result += 2
 746     return result
 747
 748
 749   minimal_postfix, remaining = remaining_dict.popitem()
 750   minimal_props, minimal_ancestor = remaining
 751   minimal_cost = calc_cost(minimal_props, minimal_ancestor)
 752   for index_postfix, (props_remaining, ancestor_remaining) in (
 753       remaining_dict.iteritems()):
 754     cost = calc_cost(props_remaining, ancestor_remaining)
 755     if cost < minimal_cost:
 756       minimal_cost = cost
 757       minimal_postfix = index_postfix
 758       minimal_props = props_remaining
 759       minimal_ancestor = ancestor_remaining
 760
 761
 762   props = frozenset(minimal_props), (minimal_postfix, frozenset(), frozenset())
 763   return False, kind, minimal_ancestor, props
 764
 765
 766
 767
 768
 769
 770 def IndexYamlForQuery(kind, ancestor, props):
 771   """Return the composite index definition YAML needed for a query.
 772
 773   Given a query, the arguments for this method can be computed with:
 774     _, kind, ancestor, props = datastore_index.CompositeIndexForQuery(query)
 775     props = datastore_index.GetRecommendedIndexProperties(props)
 776
 777   Args:
 778     kind: the kind or None
 779     ancestor: True if this is an ancestor query, False otherwise
 780     props: tuples of the form (name, direction) where:
 781         name - a property name;
 782         direction - datastore_pb.Query_Order.ASCENDING or ...DESCENDING;
 783
 784   Returns:
 785     A string with the YAML for the composite index needed by the query.
 786   """
 787
 788
 789   serialized_yaml = []
 790   serialized_yaml.append('- kind: %s' % kind)
 791   if ancestor:
 792     serialized_yaml.append('  ancestor: yes')
 793   if props:
 794     serialized_yaml.append('  properties:')
 795     for name, direction in props:
 796       serialized_yaml.append('  - name: %s' % name)
 797       if direction == DESCENDING:
 798         serialized_yaml.append('    direction: desc')
 799   return '\n'.join(serialized_yaml)
 800
 801
 802 def IndexXmlForQuery(kind, ancestor, props):
 803   """Return the composite index definition XML needed for a query.
 804
 805   Given a query, the arguments for this method can be computed with:
 806     _, kind, ancestor, props = datastore_index.CompositeIndexForQuery(query)
 807     props = datastore_index.GetRecommendedIndexProperties(props)
 808
 809   Args:
 810     kind: the kind or None
 811     ancestor: True if this is an ancestor query, False otherwise
 812     props: tuples of the form (name, direction) where:
 813         name - a property name;
 814         direction - datastore_pb.Query_Order.ASCENDING or ...DESCENDING;
 815
 816   Returns:
 817     A string with the XML for the composite index needed by the query.
 818   """
 819
 820
 821   serialized_xml = []
 822   serialized_xml.append('  <datastore-index kind="%s" ancestor="%s">'
 823                         % (kind, 'true' if ancestor else 'false'))
 824   for name, direction in props:
 825     serialized_xml.append('    <property name="%s" direction="%s" />'
 826                           % (name, 'asc' if direction == ASCENDING else 'desc'))
 827   serialized_xml.append('  </datastore-index>')
 828   return '\n'.join(serialized_xml)
 829
 830
 831 def IndexDefinitionToProto(app_id, index_definition):
 832   """Transform individual Index definition to protocol buffer.
 833
 834   Args:
 835     app_id: Application id for new protocol buffer CompositeIndex.
 836     index_definition: datastore_index.Index object to transform.
 837
 838   Returns:
 839     New entity_pb.CompositeIndex with default values set and index
 840     information filled in.
 841   """
 842   proto = entity_pb.CompositeIndex()
 843
 844   proto.set_app_id(app_id)
 845   proto.set_id(0)
 846   proto.set_state(entity_pb.CompositeIndex.WRITE_ONLY)
 847
 848   definition_proto = proto.mutable_definition()
 849   definition_proto.set_entity_type(index_definition.kind)
 850   definition_proto.set_ancestor(index_definition.ancestor)
 851
 852   if index_definition.properties is not None:
 853     for prop in index_definition.properties:
 854       prop_proto = definition_proto.add_property()
 855       prop_proto.set_name(prop.name)
 856
 857       if prop.mode == 'geospatial':
 858         prop_proto.set_mode(entity_pb.Index_Property.GEOSPATIAL)
 859       elif prop.mode == 'segment':
 860         prop_proto.set_mode(entity_pb.Index_Property.SEGMENT)
 861       elif prop.IsAscending():
 862         prop_proto.set_direction(entity_pb.Index_Property.ASCENDING)
 863       else:
 864         prop_proto.set_direction(entity_pb.Index_Property.DESCENDING)
 865
 866   return proto
 867
 868
 869 def IndexDefinitionsToProtos(app_id, index_definitions):
 870   """Transform multiple index definitions to composite index records
 871
 872   Args:
 873     app_id: Application id for new protocol buffer CompositeIndex.
 874     index_definition: A list of datastore_index.Index objects to transform.
 875
 876   Returns:
 877     A list of tranformed entity_pb.Compositeindex entities with default values
 878     set and index information filled in.
 879   """
 880   return [IndexDefinitionToProto(app_id, index)
 881           for index in index_definitions]
 882
 883
 884 def ProtoToIndexDefinition(proto):
 885   """Transform individual index protocol buffer to index definition.
 886
 887   Args:
 888     proto: An instance of entity_pb.CompositeIndex to transform.
 889
 890   Returns:
 891     A new instance of datastore_index.Index.
 892   """
 893   properties = []
 894   proto_index = proto.definition()
 895   for prop_proto in proto_index.property_list():
 896     prop_definition = Property(name=prop_proto.name())
 897
 898     if prop_proto.mode() == entity_pb.Index_Property.GEOSPATIAL:
 899       prop_definition.mode = 'geospatial'
 900     elif prop_proto.mode() == entity_pb.Index_Property.SEGMENT:
 901       prop_definition.mode = 'segment'
 902     elif prop_proto.direction() == entity_pb.Index_Property.DESCENDING:
 903       prop_definition.direction = 'desc'
 904     elif prop_proto.direction() == entity_pb.Index_Property.ASCENDING:
 905       prop_definition.direction = 'asc'
 906
 907     properties.append(prop_definition)
 908
 909   index = Index(kind=proto_index.entity_type(), properties=properties)
 910   if proto_index.ancestor():
 911     index.ancestor = True
 912   return index
 913
 914
 915 def ProtosToIndexDefinitions(protos):
 916   """Transform multiple index protocol buffers to index definitions.
 917
 918   Args:
 919     A list of entity_pb.Index records.
 920   """
 921   return [ProtoToIndexDefinition(definition) for definition in protos]